{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math, random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.autograd as autograd \n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.1.0\n"
     ]
    }
   ],
   "source": [
    "print(torch.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3>Use Cuda</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "Variable = lambda *args, **kwargs: autograd.Variable(*args, **kwargs).cuda() if USE_CUDA else autograd.Variable(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Replay Buffer</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "\n",
    "class ReplayBuffer(object):\n",
    "    def __init__(self, capacity):\n",
    "        self.buffer = deque(maxlen=capacity)\n",
    "    \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        state      = np.expand_dims(state, 0)\n",
    "        next_state = np.expand_dims(next_state, 0)\n",
    "            \n",
    "        self.buffer.append((state, action, reward, next_state, done))\n",
    "    \n",
    "    def sample(self, batch_size):\n",
    "        state, action, reward, next_state, done = zip(*random.sample(self.buffer, batch_size))\n",
    "        return np.concatenate(state), action, reward, np.concatenate(next_state), done\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Cart Pole Environment</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_id = \"CartPole-v0\"\n",
    "env = gym.make(env_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Epsilon greedy exploration</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon_start = 1.0\n",
    "epsilon_final = 0.01\n",
    "epsilon_decay = 500\n",
    "\n",
    "epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f6f42ccbc88>]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGlNJREFUeJzt3X10XPV95/H3d0YayXqyHi3bso1sYzsxIWCigIGchCSEAG1xuyfb2G02hCSl25Zm2XR3D5z0QMv+0yTttpuNm4RNQ9qE4BCaJl5q4nYTSDcUuxY4gB9BNtiWH5D8/CDLsqTv/jFXZiyPpLE88tW99/M6R0dzf/PT6Ht15Y9/+s3v3mvujoiIxEsq7AJERKT4FO4iIjGkcBcRiSGFu4hIDCncRURiSOEuIhJDCncRkRhSuIuIxJDCXUQkhkrC+saNjY3e2toa1rcXEYmkF1988aC7N43VL7Rwb21tpb29PaxvLyISSWa2q5B+mpYREYkhhbuISAwp3EVEYkjhLiISQwp3EZEYGjPczexbZtZlZptGeN7M7Ctm1mFmr5jZdcUvU0RELkYhI/dvA7eP8vwdwILg417ga5deloiIXIoxw93d/wU4PEqXZcDfedY6oNbMZhSrwOE2vHmYL/5kG7o9oIjIyIox594C7MnZ7gzaLmBm95pZu5m1d3d3j+ubvbznKF97bgfHT/eP6+tFRJKgGOFuedryDqvd/VF3b3P3tqamMc+ezauhKgPAoVNnxvX1IiJJUIxw7wRm52zPAvYV4XXzqq8sA+Dwqb6J+hYiIpFXjHBfDXwyWDWzFDjm7vuL8Lp5NVQOjdwV7iIiIxnzwmFm9gRwC9BoZp3Aw0ApgLt/HVgD3Al0AD3APRNVLEB9EO4auYuIjGzMcHf3FWM878AfFK2iMSjcRUTGFrkzVMtL01Rm0hw6qXAXERlJ5MIdoK4yw2GtlhERGVEkw72hMqM3VEVERhHJcK+vzGjOXURkFBEN9zKFu4jIKCIZ7g1V2ZG7ri8jIpJfJMO9vjLDmf5BevoGwi5FRGRSimy4g9a6i4iMJJLhrksQiIiMLpLh/vbIXWvdRUTyiWS4NwRXhtRZqiIi+UUy3OurNOcuIjKaSIZ7ZSZNpiSlcBcRGUEkw93MdAkCEZFRRDLcQZcgEBEZTaTDXSN3EZH8IhvuDbrsr4jIiKIb7lVlWgopIjKCyIZ7Y1UZPX0D9PT1h12KiMikE9lwb6rOnsh08IRG7yIiw0U+3LtP9oZciYjI5BPZcG8MzlLtPqE3VUVEhotsuJ8buSvcRUQuENlwb6gsI2XQrRUzIiIXiGy4p1NGfWWZRu4iInlENtwhO++ucBcRuVCkw72puozukwp3EZHhIh/uBzVyFxG5QLTDvSo7cnf3sEsREZlUoh3u1WX09Q9yvFeXIBARyRX5cAc4qHl3EZHzFBTuZna7mW03sw4zeyDP83PM7Fkz22hmr5jZncUv9UJNVTqRSUQknzHD3czSwErgDmAxsMLMFg/r9sfAk+6+BFgO/HWxC82nUWepiojkVcjI/Xqgw913unsfsApYNqyPAzXB46nAvuKVODKN3EVE8isk3FuAPTnbnUFbrj8BPmFmncAa4A/zvZCZ3Wtm7WbW3t3dPY5yzzd1SimladOcu4jIMIWEu+VpG772cAXwbXefBdwJfMfMLnhtd3/U3dvcva2pqeniqx0mlTIadAkCEZELFBLuncDsnO1ZXDjt8hngSQB3fwEoBxqLUeBYdJaqiMiFCgn3DcACM5trZhmyb5iuHtZnN/BhADN7J9lwv/R5lwI0VZfRdVzhLiKSa8xwd/d+4D5gLbCV7KqYzWb2iJndFXT7I+B3zOxl4AngU36ZThttrimjS9MyIiLnKSmkk7uvIftGaW7bQzmPtwA3F7e0wjTXlHPo1BnODgxSmo70OVkiIkUT+TRsrinHHY3eRURyRD7cp9eUA3DgmG6ULSIyJPLh3hyE+1vHFe4iIkMiH+7Tp2rkLiIyXOTDva6ilExJSiN3EZEckQ93M6O5powDCncRkXMiH+6QfVNVI3cRkbfFItyba8p5S2epioicE4twn15TzoFjvbqXqohIIBbh3lxTzumzA7qXqohIIB7hPlVr3UVEcsUi3HWWqojI+eIV7hq5i4gAMQn3aTXZe6l2KdxFRICYhHt5aZq6ilKN3EVEArEId8iumNGcu4hIVmzCvaV2CvuOKtxFRCBG4T6zdgp7j54OuwwRkUkhVuF+7PRZTp7RiUwiIrEJ95a6KQDs1+hdRCRG4V6bXeveqXAXEYlPuM+szY7c9yncRUTiE+7TqsspSRl7jyjcRURiE+7plDF9arlG7iIixCjcQWvdRUSGxC7ctdZdRCRm4T6zdgoHjvfSPzAYdikiIqGKVbi31E1hYNB564TupyoiyRarcNdySBGRrFiF+9CJTFoOKSJJV1C4m9ntZrbdzDrM7IER+vymmW0xs81m9r3illmYoZG73lQVkaQrGauDmaWBlcBHgE5gg5mtdvctOX0WAA8CN7v7ETObNlEFj6YiU0JdRSmdGrmLSMIVMnK/Huhw953u3gesApYN6/M7wEp3PwLg7l3FLbNwc+or6DzSE9a3FxGZFAoJ9xZgT852Z9CWayGw0MyeN7N1ZnZ7sQq8WLPrK9h9WOEuIslWSLhbnjYftl0CLABuAVYA3zSz2gteyOxeM2s3s/bu7u6LrbUgVzRUsPfIaa11F5FEKyTcO4HZOduzgH15+vzY3c+6+xvAdrJhfx53f9Td29y9rampabw1j2pOfQX9g85+3U9VRBKskHDfACwws7lmlgGWA6uH9fkR8EEAM2skO02zs5iFFmpOfSUAuw5pakZEkmvMcHf3fuA+YC2wFXjS3Teb2SNmdlfQbS1wyMy2AM8C/9XdD01U0aOZ01ABoHl3EUm0MZdCArj7GmDNsLaHch478PngI1TTa8rJpFPsOnwq7FJEREITqzNUIXtd91l1U9ijkbuIJFjswh2yUzOalhGRJItnuNdXsOtQD9nZIhGR5IltuJ/o7efY6bNhlyIiEorYhjtoOaSIJFc8w13LIUUk4eIZ7udG7loOKSLJFMtwr8iUML2mnJ0HFe4ikkyxDHeAeU2V7OxWuItIMsU23Oc2VrKz+6SWQ4pIIsU23Oc1VXG8t59Dp/rCLkVE5LKLcbhnrw6pqRkRSaLYhvv8xioA3jh4MuRKREQuv9iGe0vdFDIlKY3cRSSRYhvu6ZTR2lDBDoW7iCRQbMMdYF5jFTs1LSMiCRTvcG+qZPehHs7qZtkikjAxD/cq+gddN+4QkcSJebhrOaSIJFOsw31+U3Y55OtdmncXkWSJdbhPnVLKjKnlvPbWibBLERG5rGId7gALm6vZfkDhLiLJEvtwXzS9mo7uk/RrxYyIJEj8w725mr7+QXZpxYyIJEj8w316NYCmZkQkUWIf7ldOq8JM4S4iyRL7cC8vTdPaUKkVMyKSKLEPd4CFzVVsV7iLSIIkItwXNVfz5sFT9J4dCLsUEZHLIhHhvnB6NYMOHTpTVUQSIhHh/s4ZNQBs3X885EpERC6PgsLdzG43s+1m1mFmD4zS72Nm5mbWVrwSL93chkoqM2k271O4i0gyjBnuZpYGVgJ3AIuBFWa2OE+/auBzwPpiF3mpUilj8cwaNu09FnYpIiKXRSEj9+uBDnff6e59wCpgWZ5+/x34EtBbxPqK5qqZU9my/zgDgx52KSIiE66QcG8B9uRsdwZt55jZEmC2uz9dxNqK6l0tU+npG+CNg7q2u4jEXyHhbnnazg1/zSwF/CXwR2O+kNm9ZtZuZu3d3d2FV1kE72rJvqm6eZ+mZkQk/goJ905gds72LGBfznY18C7gOTN7E1gKrM73pqq7P+rube7e1tTUNP6qx2F+UxWZkpTm3UUkEQoJ9w3AAjOba2YZYDmweuhJdz/m7o3u3ururcA64C53b5+QisepNJ3indOr2bRXK2ZEJP7GDHd37wfuA9YCW4En3X2zmT1iZndNdIHFdFXLVDbtO4a73lQVkXgrKaSTu68B1gxre2iEvrdcelkT410zp/K99bvZfbiHKxoqwy5HRGTCJOIM1SHXzq4F4Jd7joZciYjIxEpUuC9srqIik+alXUfCLkVEZEIlKtxL0imumVXLS7s1cheReEtUuAMsmVPL1v3HOd2ny/+KSHwlLtyvm1NH/6Dzqta7i0iMJS7cr52TfVN1427Nu4tIfCUu3BuryriioYKXFO4iEmOJC3eAJbOzb6rqZCYRiatEhvt7rqij+8QZ9hw+HXYpIiITIpHhvnReAwDrdh4KuRIRkYmRyHC/cloVjVUZXlC4i0hMJTLczYwb5jWwbuchzbuLSCwlMtwhOzWz/1gvuw/3hF2KiEjRJTbcb5xXD2jeXUTiKbHhPr8pO+++bufhsEsRESm6xIb70Lz7Czs07y4i8ZPYcAd435WNHDjey+tdJ8MuRUSkqBId7h9YmL1J98+3d4dciYhIcSU63GfWTmFhcxXPvdYVdikiIkWV6HAHuGXRNDa8cYRTZ/rDLkVEpGgSH+4fWNhE38AgL+zQkkgRiY/Eh3tbax0VmbSmZkQkVhIf7mUlaW6a38Cz27q1JFJEYiPx4Q5w2+Lp7D16ms37joddiohIUSjcgVsXN5My+MmmA2GXIiJSFAp3oL4yww1zG3hm0/6wSxERKQqFe+COq6ezo/sUHV0nwi5FROSSKdwDty2eDsAzr2pqRkSiT+EemD61nCVzavnHVzU1IyLRp3DP8evXtrDtwAm27teqGRGJNoV7jl+7ZiYlKeMfNu4NuxQRkUtSULib2e1mtt3MOszsgTzPf97MtpjZK2b2UzO7ovilTrz6ygy3LJrGjzbuZWBQJzSJSHSNGe5mlgZWAncAi4EVZrZ4WLeNQJu7vxt4CvhSsQu9XP7ddS10nTjD8x0Hwy5FRGTcChm5Xw90uPtOd+8DVgHLcju4+7PuPnSn6XXArOKWefl86B3TqCkv4akXO8MuRURk3AoJ9xZgT852Z9A2ks8Az1xKUWEqL03zG0ta+MmmAxw6eSbsckRExqWQcLc8bXknpM3sE0Ab8OURnr/XzNrNrL27e/Le/egTS6+gb2CQJ9s1eheRaCok3DuB2Tnbs4B9wzuZ2a3AF4C73D3vkNfdH3X3Nndva2pqGk+9l8WC5mqWzqvn8fW79MaqiERSIeG+AVhgZnPNLAMsB1bndjCzJcA3yAZ7LC6M/h+WttJ55DQ/13XeRSSCxgx3d+8H7gPWAluBJ919s5k9YmZ3Bd2+DFQBPzCzX5rZ6hFeLjJuu6qZadVlPPb8m2GXIiJy0UoK6eTua4A1w9oeynl8a5HrCl1pOsWn3zeXP3tmG692HuPqWVPDLklEpGA6Q3UUv33DHKrLS/jr5zrCLkVE5KIo3EdRXV7KJ2+8gp9sPkBH18mwyxERKZjCfQz33DyXTDql0buIRIrCfQyNVWXcfVMr/7BxL9sP6EYeIhINCvcC/P4t86kqK+HLa7eFXYqISEEU7gWorcjwHz8wn/+7tYsNbx4OuxwRkTEp3Av06Zvn0lxTxp/+n806a1VEJj2Fe4GmZNL88a8sZtPe43x33a6wyxERGZXC/SL86rtn8L4rG/nztdvpOt4bdjkiIiNSuF8EM+ORZVdxpn+Qh1dvxl3TMyIyOSncL9K8piru/8gCntl0gB++pHutisjkpHAfh999/3yub63n4dWb2XO4Z+wvEBG5zBTu45BOGX/xm9cA8LlVGznTPxByRSIi51O4j9Ps+gq+/LF3s3H3UR7+sebfRWRyUbhfgjuunsEffHA+qzbs4bvrd4ddjojIOQVdz11G9vmPLGLr/hM8/ONNTKsu46NXTQ+7JBERjdwvVTplfPW3lnDN7Fr+8ImN/OuOg2GXJCKicC+GikwJj33qvbQ2VPDZv23n+Q4FvIiES+FeJLUVGb772RuYXVfBPY9t4J82Hwi7JBFJMIV7EU2rLuf7v7uUxTNr+L3HX+Jbv3hDq2hEJBQK9yKrrcjw+Gdv4MPvmMYjT2/hv/zgFXrPah28iFxeCvcJUFlWwtc/8R7uv3UBf/9SJ8u++jyb9x0LuywRSRCF+wRJpYz7b13IY/e8l8M9ffz6yuf5Xz99XWezishloXCfYB9cNI1/uv/93HbVdP7in1/jo3/5L/xs21thlyUiMadwvwzqKjOs/K3r+PY97yWVMj797XZWPLqOF3YcCrs0EYkpC2s1R1tbm7e3t4fyvcPU1z/Id9ft4ms/30H3iTNc31rPPTe3cuviZkrT+r9WREZnZi+6e9uY/RTu4eg9O8Cqf9vN//5/b7D36GmmVZfx8ffOZtm1M7lyWnXY5YnIJKVwj4iBQefZbV08vn4Xz73WjTssaq7mzqtn8KF3TOOqmTWkUhZ2mSIySSjcI+it47088+p+/vHV/bTvOoI71FWUctP8Rm6c38C1s2tZNL1a0zciCaZwj7iuE738a8chftFxkF+8fpADwQ25MyUprppZw9UtU7lyWhXzm7IfzTVlmGmELxJ3CvcYcXf2HD7Ny51HeaXzKC/vOcaW/cc5eab/XJ+qshJm1U1hZu0UZtaWM2Nq9nNzdTl1lRnqKzPUVpRSVpIOcU9E5FIVGu4FXc/dzG4H/ieQBr7p7n827Pky4O+A9wCHgI+7+5sXW7TkZ2bMaahgTkMFv3bNTCAb+F0nzrCj6yQd3SfZ0XWSziOn2Xesl5d2H+Foz9m8r1VVVkJdZSl1FRkqMyVUlpVQWZbOfs4MfS6hoixNWUma0rRRVpIiU5Iik85uZ4LtspIUpensRzplpMxIp4y0GakU59qG2lOG/roQuUzGDHczSwMrgY8AncAGM1vt7ltyun0GOOLuV5rZcuCLwMcnomDJMjOaa8pprinnpisbL3i+p6+f/cd6eet4L0d7znL4VB9HTvVxpOcsR3r6OHyqj1Nn+tl79DQ9ff2cOtPPqTMDnJ7g6+CkjAv+I7Ag9Idy34L9G/pvINtu5x7ntlvedsv5utH7Tbr/aiZZQZOsnEk3OBhvNZ/78IJzA7WJUsjI/Xqgw913ApjZKmAZkBvuy4A/CR4/BXzVzMx1ScTQVGRKzs3HX4yBQaenr5+evgH6+gc50z/I2YFB+voH6Rv6PKz97MAgA+4MDjoDg86Ak33s2W13Z2CQt/uc19dx59zVMx2y2wTbDkO/RNkuOe3BE47nPD7/6znv6/2815psv5yT7Z/L5KqGSVeQX0JBU6eUFrGS/AoJ9xZgT852J3DDSH3cvd/MjgENgO5aETHplFFdXkp1+cT/8onIxClkTV2+vzyG/5dVSB/M7F4zazez9u7u7kLqExGRcSgk3DuB2Tnbs4B9I/UxsxJgKnB4+Au5+6Pu3ububU1NTeOrWERExlRIuG8AFpjZXDPLAMuB1cP6rAbuDh5/DPiZ5ttFRMIz5px7MId+H7CW7FLIb7n7ZjN7BGh399XA3wDfMbMOsiP25RNZtIiIjK6gde7uvgZYM6ztoZzHvcC/L25pIiIyXrpIiYhIDCncRURiSOEuIhJDoV04zMy6gV3j/PJGkneClPY5GbTPyXAp+3yFu4+5ljy0cL8UZtZeyFXR4kT7nAza52S4HPusaRkRkRhSuIuIxFBUw/3RsAsIgfY5GbTPyTDh+xzJOXcRERldVEfuIiIyisiFu5ndbmbbzazDzB4Iu57xMrPZZvasmW01s81m9p+C9noz+2czez34XBe0m5l9JdjvV8zsupzXujvo/7qZ3T3S95wszCxtZhvN7Olge66ZrQ/q/35wgTrMrCzY7gieb815jQeD9u1m9tFw9qQwZlZrZk+Z2bbgeN8Y9+NsZv85+L3eZGZPmFl53I6zmX3LzLrMbFNOW9GOq5m9x8xeDb7mK2YXeRsqd4/MB9kLl+0A5gEZ4GVgcdh1jXNfZgDXBY+rgdeAxcCXgAeC9geALwaP7wSeIXvt/KXA+qC9HtgZfK4LHteFvX9j7Pvnge8BTwfbTwLLg8dfB34vePz7wNeDx8uB7wePFwfHvgyYG/xOpMPer1H292+BzwaPM0BtnI8z2Zv3vAFMyTm+n4rbcQbeD1wHbMppK9pxBf4NuDH4mmeAOy6qvrB/QBf5w7wRWJuz/SDwYNh1FWnffkz2PrXbgRlB2wxge/D4G8CKnP7bg+dXAN/IaT+v32T7IHs/gJ8CHwKeDn5xDwIlw48x2SuR3hg8Lgn62fDjnttvsn0ANUHQ2bD22B5n3r4zW31w3J4GPhrH4wy0Dgv3ohzX4LltOe3n9SvkI2rTMvlu+dcSUi1FE/wZugRYDzS7+36A4PO0oNtI+x61n8lfAf8NGAy2G4Cj7t4fbOfWf97tG4Gh2zdGaZ/nAd3AY8FU1DfNrJIYH2d33wv8ObAb2E/2uL1IvI/zkGId15bg8fD2gkUt3Au6nV+UmFkV8PfA/e5+fLSuedp8lPZJx8x+Fehy9xdzm/N09TGei8w+kx2JXgd8zd2XAKfI/rk+ksjvczDPvIzsVMpMoBK4I0/XOB3nsVzsPl7yvkct3Au55V9kmFkp2WB/3N1/GDS/ZWYzgudnAF1B+0j7HqWfyc3AXWb2JrCK7NTMXwG1lr09I5xf/0i3b4zSPncCne6+Pth+imzYx/k43wq84e7d7n4W+CFwE/E+zkOKdVw7g8fD2wsWtXAv5JZ/kRC88/03wFZ3/x85T+XesvBusnPxQ+2fDN51XwocC/7sWwvcZmZ1wYjptqBt0nH3B919lru3kj12P3P33waeJXt7Rrhwn/PdvnE1sDxYZTEXWED2zadJx90PAHvMbFHQ9GFgCzE+zmSnY5aaWUXwez60z7E9zjmKclyD506Y2dLgZ/jJnNcqTNhvSIzjDYw7ya4s2QF8Iex6LmE/3kf2z6xXgF8GH3eSnWv8KfB68Lk+6G/AymC/XwXacl7r00BH8HFP2PtW4P7fwturZeaR/UfbAfwAKAvay4PtjuD5eTlf/4XgZ7Gdi1xFEMK+Xgu0B8f6R2RXRcT6OAN/CmwDNgHfIbviJVbHGXiC7HsKZ8mOtD9TzOMKtAU/vx3AVxn2pvxYHzpDVUQkhqI2LSMiIgVQuIuIxJDCXUQkhhTuIiIxpHAXEYkhhbuISAwp3EVEYkjhLiISQ/8fSrxxVFxHn1MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([epsilon_by_frame(i) for i in range(10000)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Deep Q Network</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions):\n",
    "        super(DQN, self).__init__()\n",
    "        \n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(env.observation_space.shape[0], 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, env.action_space.n)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.layers(x)\n",
    "    \n",
    "    def act(self, state, epsilon):\n",
    "        if random.random() > epsilon:\n",
    "            with torch.no_grad():\n",
    "                state   = Variable(torch.FloatTensor(state).unsqueeze(0))\n",
    "            q_value = self.forward(state)\n",
    "            action  = q_value.max(1)[1].data[0]\n",
    "        else:\n",
    "            action = random.randrange(env.action_space.n)\n",
    "        return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = DQN(env.observation_space.shape[0], env.action_space.n)\n",
    "\n",
    "if USE_CUDA:\n",
    "    model = model.cuda()\n",
    "    \n",
    "optimizer = optim.Adam(model.parameters())\n",
    "\n",
    "replay_buffer = ReplayBuffer(1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Computing Temporal Difference Loss</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_td_loss(batch_size):\n",
    "    state, action, reward, next_state, done = replay_buffer.sample(batch_size)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        state      = Variable(torch.FloatTensor(np.float32(state)))\n",
    "        next_state = Variable(torch.FloatTensor(np.float32(next_state)))\n",
    "        action     = Variable(torch.LongTensor(action))\n",
    "        reward     = Variable(torch.FloatTensor(reward))\n",
    "        done       = Variable(torch.FloatTensor(done))\n",
    "\n",
    "    q_values      = model(state)\n",
    "    next_q_values = model(next_state)\n",
    "\n",
    "    q_value          = q_values.gather(1, action.unsqueeze(1)).squeeze(1)\n",
    "    next_q_value     = next_q_values.max(1)[0]\n",
    "    expected_q_value = reward + gamma * next_q_value * (1 - done)\n",
    "    \n",
    "    loss = (q_value - Variable(expected_q_value.data)).pow(2).mean()\n",
    "        \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot(frame_idx, rewards, losses):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, np.mean(rewards[-10:])))\n",
    "    plt.plot(rewards)\n",
    "    plt.subplot(132)\n",
    "    plt.title('loss')\n",
    "    plt.plot(losses)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Training</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAv4AAAE/CAYAAAA+Occ1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xuco3V5///XlWTOez6y7LIsIFWOAq5CPStWDtKirVasD6WW74/a6k9tbRUPP+GrtV9rrQdqxQcqFftVkIIUFBQRQUDlsJwWlhV3WfbELruzM7s752SSXL8/7juZzEwyk5nJ5E4m7+fjMY9JPved5JPZ2cyVK9fn+pi7IyIiIiIic1ss6gmIiIiIiMjsU+AvIiIiItIAFPiLiIiIiDQABf4iIiIiIg1Agb+IiIiISANQ4C8iIiIi0gAU+M8RZvZiM3vMzHrN7ENRz0dml5ltN7M3RT0PEZG5RK+tMtcp8J87Pgbc4+7z3f3KqCczlpldbWbPmFnWzP6yyPG/M7MXzOywmV1jZi0Fx9aZ2d1mNmBmvxv7ojyT2zYCM/tHM3sqfFP4nJn945jj0/75Fnmss8P7GAjv8+jZel4iIiIyNQr8546jgU2lDppZvIpzKeYJ4G+BR8ceMLNzgMuAs4F1wLHA/y445TrgMWAp8CngRjNbPtPbToWZJaZ6m0qo0OMa8F5gMXAu8EEzu6jg+Ex+voVzXQb8CPj/gCXABuCHFZi/iIiIVIAC/znAzH4JvAH4upn1mdkfmNl3zewqM7vdzPqBN5jZW8JyoB4z22VmVxTcxzozczN7X3jsoJm938xebmYbzeyQmX19zOP+lZltDs+9Y6Lsrrv/h7vfBQwVOXwx8B133+TuB4HPAX8ZPsYfAGcAl7v7oLvfBDwJ/FkFbjvZz3W7mX3czDYC/WaWMLMjzewmM+sMs+cfCs9tNbPBMPjFzD5tZmkzWxBe/ycz+2p4uZx/h0vMbCfwy3D8PWa2w8y6zOxT5cw/x92/6O6Punva3Z8BbgFeNdOfbxF/Cmxy9/929yHgCuClZvaSqcxXRCRqZtZiZl81sz3h11dzn3aa2TIz+0n4d7HbzO4zs1h47ONm9nz4CeszZnZ2tM9EZDQF/nOAu78RuA/4oLvPc/ffh4f+Avg8MB+4H+gnyPwuAt4C/I2ZvXXM3Z0JHA+8E/gqQQb4TcBJwJ+b2esAwtt9kiDYWx4+/nXTfAonEXwikPMEsNLMlobHtrl775jjJ1XgtuV4F8HPahGQBX4c3sdqgiz4R8zsnDDQfRh4XXi71wI7CAPs8Pqvwsvl/Du8DjgBOMfMTgSuAt4DHEmQmV+TO9HMXm1mh8p5MmZmwGsY+XRoJj/fsUad6+79wLNM7ectIlILPgWcBZwGvBR4BfDp8NhHgd0Ef/tWEvwtdDN7MfBB4OXuPh84B9he3WmLTEyB/9x2i7v/2t2z7j7k7ve4+5Ph9Y0Egfrrxtzmc+G5PycIUK9z9/3u/jxBcH96eN5fA//H3Te7exr4Z+C0adZ0zwMOF1zPXZ5f5Fju+PwK3LYcV7r7LncfBF4OLHf3z7p7yt23Ad8CcmUzvwJeF5bnnApcGV5vDW97H0CZ/w5XuHt/+LhvB37i7ve6e5KglCabO9Hd73f3RWU+nysI/t//Z3h9Jj/fsSrx8xYRqQXvBj4b/v3rJChxfE94bBhYBRzt7sPufp+7O5ABWoATzazJ3be7+7ORzF6kBAX+c9uuwitmdma44LLTzA4D7weWjbnNvoLLg0WuzwsvHw18Lfyo8xDQTVBLvnoa8+wDFhRcz13uLXIsdzyXoZ7JbctR+DM8Gjgy95zD5/1JgowPBIH/6wlKZ54E7iQI6M8Ctrr7ASj736HwcY8svB5m0rum8BwIH/eDBJ80vCV8AwEz+/mOVYmft4hILTiS4FPbnB3hGMC/AluBn5vZNjO7DMDdtwIfIUiw7Dez683sSERqiAL/uc3HXP8BcCtwlLsvBL5JEKxPxy7gr919UcFXm7v/Zhr3tYngo9SclwL73L0rPHasmc0fc3xTBW5bjsKf4S7guTHPeb67nx8e/w3wYuBtwK/c/WlgLUE5z68K7qecf4fCx90LHJW7YmbtBOU+ZTOzvyJcpOvuuwsOzeTnO9aoc82sAziOqf28RURqwR6CZE/O2nAMd+9194+6+7HAHwN/n6vld/cfuPurw9s68C/VnbbIxBT4N5b5QLe7D5nZKwjWAEzXN4FPmNlJAGa20MzeUepkM2sOS14MaAoXw+Z+/74HXGJmJ5rZYoI6yu8ChOsVHgcuD2/zNoIympsqcNupegjoCRdvtZlZ3MxONrOXh483ADwCfICRQP83BGVRhYH/VP8dbgQuCGv5m4HPMoX/u2b2boJSrD8Ky5PyZvLzLeJm4GQz+7Pw3/ozwEZ3/125cxURqRHXAZ82s+Vh04bPAP8XwMwuMLMXhWumeghKfDIW7KfzxnAR8BDBp+SZiOYvUpQC/8byt8BnzayX4EXshunekbvfTJDJuN7MeoCngPMmuMnPCV4EXwlcHV5+bXhfPwO+CNxN8HHqDuDygtteBKwHDgJfAN4e1lzO6LZm9m4zKzsb7e4ZguzOacBzwAHg28DCgtN+BTQRvEnIXZ8P3FtwzpT+Hdx9E8GbiR8QZP8PEiwsI3werzGzvgnu4p8IPiF42IKuT31m9s2C49P++ZrZpvCNBeFt/oxgQflBgoXihW1DRUTqxT8RtCTeSFC6+Wg4BkEDjF8QlDf+FviGu99DUN//BYK/DS8AKwjKQUVqhgXrUUREREREZC5Txl9EREREpAEo8BcRERERaQAK/EVEREREGoACfxERERGRBqDAX0RERESkASSingDAsmXLfN26dVFPQ0SkJj3yyCMH3H151POIkv5OiIgUN5W/ETUR+K9bt44NGzZEPQ0RkZpkZjuinkPU9HdCRKS4qfyNUKmPiIiIiEgDUOAvIiIiItIAFPiLiIiIiDQABf4iIiIiIg1Agb+IiIiISANQ4C8iIiIi0gAU+IuIiIiINIBJA38zO8rM7jazzWa2ycw+HI4vMbM7zWxL+H1xOG5mdqWZbTWzjWZ2xmw/CRERERERmVg5Gf808FF3PwE4C/iAmZ0IXAbc5e7HA3eF1wHOA44Pvy4Frqr4rEVEREREZEom3bnX3fcCe8PLvWa2GVgNXAi8PjztWuAe4OPh+Pfc3YEHzGyRma0K70ekbPf+vpNjl3ewZnF71FMZ5UBfkrs27yPrUc9E6snL1y3hRSvmRT0NqSNb9/fS3pzgyEVtUU9FROaISQP/Qma2DjgdeBBYmQvm3X2vma0IT1sN7Cq42e5wbFTgb2aXEnwiwNq1a6cxdZnLdnUP8N5rHgLgrGOX8KdnrOH8U1Yxr2VKv7Kz4trfbOfff7k16mlInfnCn56iwF+m5E1fvheA7V94S8QzEZG5ouwoyszmATcBH3H3HjMreWqRsXG5UXe/GrgaYP369cqdyiiHBoYBOPekI3hmXy8fu3Ej1z20k5v/9lURzwwGUxnamuLc/Q+vj3oqUkcWtEX/plVERBpbWX+JzKyJIOj/vrv/KBzelyvhMbNVwP5wfDdwVMHN1wB7KjVhaQz9qTQA7/3Do/nD45bywR88xuO7DkU8q0A66zTFjSMWtkY9FREREZGyldPVx4DvAJvd/csFh24FLg4vXwzcUjD+3rC7z1nAYdX3y1QNpjIAtDXHMTM6WuJkaqSoPutOPFbyEy8RERGRmlROxv9VwHuAJ83s8XDsk8AXgBvM7BJgJ/CO8NjtwPnAVmAAeF9FZywNYSAM/Nubg1/ReCxGxmsj8E9nnXhMW2CIiIhIfSmnq8/9FK/bBzi7yPkOfGCG85IGlyv1aW+OA5CIWc1k/DMZJ6GMv4iIiNQZpS2lJg3mM/5B4B+PGelMNsop5WVU6iMiIiJ1SIG/1KRcxr+jJVfqYzXTNz+TVeAvIiIi9UeBv9SkwVQGM2hJBL+iiZiRztZGxj+dVamPiIiI1B8F/lKTBlIZ2puCjj4AsRqq8c9mnZgCfxEREakzCvylJg2k0rQX7NJbS4t709msMv4iIiJSdxT4S00aSGXyC3thpMY/WwPBv2r8RUREpB4p8Jea1J/M5Hv4A8TDkp9a6OWfVuAvIiIidUiBv9SkweH06Ix/PAz8lfEXERERmRYF/lKTxpb65GrqayXwV42/iIiI1BsF/lKTBpKjA/9YWOqTroHAX6U+IiIiUo8U+EtNGhhOj6rxz2XYa2Fxb1aBv4iIiNQhBf5Sk8Zm/OPx4Fe1djL++q8jIiIi9UXRi9Skce08TTX+IiIiIjOhwF9qTjbrDA5naCtS6lML7TwzWc+vORARERGpFwr8peYMDmcA6BizgRdAJlMbgb8y/iIiIlJvFPhLzRlIBYH/2J17AdLZbCRzKpTOZvP7CoiIiIjUCwX+UnMGUmmA0Tv35rr61EipT1ylPiIiIlJnFPhLzSmW8U/EaqePf8ZV6iMiIiL1R4G/1Jxc4N9WuIFXLvCvhRr/jPr4i4iISP1R4C81J1fq09FSZAOvGij10c69MteZ2TVmtt/MnioYW2Jmd5rZlvD74nDczOxKM9tqZhvN7IyC21wcnr/FzC4uGH+ZmT0Z3uZKM9XOiYhUgwJ/qTn5jH9TscW90Qf+WVfgL3Ped4Fzx4xdBtzl7scDd4XXAc4Djg+/LgWuguCNAnA5cCbwCuDy3JuF8JxLC2439rFERGQWKPCXmjMYBv6FGf98O88aCPzTaucpc5y73wt0jxm+ELg2vHwt8NaC8e954AFgkZmtAs4B7nT3bnc/CNwJnBseW+Duv3V3B75XcF8iIjKLFPhLzenPd/Up0se/BgL/oMZf/3Wk4ax0970A4fcV4fhqYFfBebvDsYnGdxcZFxGRWTZp9FKi1vOHZvZ4+LXdzB4Px9eZ2WDBsW/O5uRlbhossrg3EQbaNRH4uxNX3C+SU+zjL5/G+Pg7NrvUzDaY2YbOzs4ZTFFERKC8jP93GVN/6e7vdPfT3P004CbgRwWHn80dc/f3V26q0ij6k2E7z1E1/sH3WqjxDxb3KvKXhrMvLNMh/L4/HN8NHFVw3hpgzyTja4qMj+PuV7v7endfv3z58oo8CRGRRjZp9FKi1hMIujkAfw5cV+F5SQMbGE7TnIiRKEir5wLtbA0E/hnV+EtjuhXIdea5GLilYPy9YXefs4DDYSnQHcCbzWxxuKj3zcAd4bFeMzsr/Bvy3oL7EhGRWZSY/JQJvQbY5+5bCsaOMbPHgB7g0+5+3wwfQxrMYCpDR0GZD9TOBl7uTibr+X0FROYiM7sOeD2wzMx2E3Tn+QJwg5ldAuwE3hGefjtwPrAVGADeB+Du3Wb2OeDh8LzPunsuifQ3BJ8mtwE/Db9ERGSWzTTwfxejs/17gbXu3mVmLwP+x8xOcveesTc0s0sJ2rmxdu3aGU5D5pL+ZIb25tG/mjHLLe7NRjGlvNz7DmX8ZS5z93eVOHR2kXMd+ECJ+7kGuKbI+Abg5JnMUUREpm7ahcpmlgD+FPhhbszdk+7eFV5+BHgW+INit1ftppQyOJwetbAXIBHPBf5RzGhEOnzjoT7+IiIiUm9mskLxTcDv3D3fls3MlptZPLx8LMHGLNtmNkVpNP3J8aU+Ixt4RRv557oKKfAXERGRelNOO8/rgN8CLzaz3WF9J8BFjF/U+1pgo5k9AdwIvL+gplOkLIOpzPiMf4308c89vkp9REREpN5MWuNfqtbT3f+yyNhNBO09RaZtYDjNyvmto8ZyNf5RL+5Vxl9ERETqlZqRS80ZSBbJ+Ic1/lG380wr4y8iIiJ1SoG/1JyBVIb2kjX+0Qb+uTceaucpIiIi9UaBv9Sc/lR6XDvPuNVGjb8y/iIiIlKvFPhLzRkskvFPhDv3Rh34j9T467+OiIiI1BdFL1JTUuks6azT0TIm4x+vjYz/SOAf6TREREREpkzhi9SUgVQagLamMTX+NdLVJ62Mv4iIiNQpRS9SUwZSGYCSi3uzXhsZf9X4i4iISL1R4C81JZfxbx9T6pMLtNOZqDP+wc7BuX0FREREROqFAn+pKfmM/5hSn1h+595s1edUKPfwyviLiIhIvVHgLzUlH/i3xMcdS8SMTMSlPrmMf26xsYiIiEi9UOAvNSVf6jOmjz8Edf5RL+7Nd/VRqY+IiIjUGQX+UlNKLe6FIPDPRFzjr8W9IiIiUq8U+EtNGUhOEvjXSFefuAJ/ERERqTMK/KWmTFTqk4hZ5Bt45UqNEqrxFxERkTqjwF9qysDwxBn/yGv8w08c1M5TRERE6o0Cf6kpA8kMMYOWxPhfzXjMyEYd+GdyNf76ryMiIiL1RdGL1JSBVIb25gRWJKOeiMUiz/inVeMvIiIidUqBv9SUgVS6aJkPQCxG5DX+WtwrIiIi9UqBv9SUIONfPPBPxGLRB/6uwF9ERETqkwJ/qSm5Up9i4jXQ1ScT7tyrPv4iIiJSbxT4S02ZqNQnbkY6DLyjks4o4y8iIiL1SYG/1JSBVIa2UoF/zMhEG/eTVamPiIiI1CkF/lJTBlJpOkqU+iTili+1iUp+Ay8F/iIiIlJnJg38zewaM9tvZk8VjF1hZs+b2ePh1/kFxz5hZlvN7BkzO2e2Ji5z00SLe2NWAxt4qauPiIiI1KlyMv7fBc4tMv4Vdz8t/LodwMxOBC4CTgpv8w0zKx7FiRQxmMrQ3lKqq4/lS22iosBfRERE6tWkgb+73wt0l3l/FwLXu3vS3Z8DtgKvmMH8pMH0p9ITdvXJLa6NigJ/ERERqVczqfH/oJltDEuBFodjq4FdBefsDsdEJpXNOkPDWdqaJlrcWxs79yZiWh4jIiIi9WW60ctVwHHAacBe4N/C8WJp0KKRmpldamYbzGxDZ2fnNKchc8ngcAaAjhKlPvGY5TfQikrujYfifhEREak30wpf3H2fu2fcPQt8i5Fynt3AUQWnrgH2lLiPq919vbuvX758+XSmIXNMfyoNQFuprj41kPHPKOMvIiIidWpa0YuZrSq4+jYg1/HnVuAiM2sxs2OA44GHZjZFaRSDqTDjP0Ef/6hr/HOlPirxFxERkXpTPLVawMyuA14PLDOz3cDlwOvN7DSCMp7twF8DuPsmM7sBeBpIAx9w98zsTF3mmv5k8KtScufemujqkyUeM8wU+YuIiEh9mTTwd/d3FRn+zgTnfx74/EwmJY1pcHiyUp9YDfTxV0cfaWxm9nfA/yJI/DwJvA9YBVwPLAEeBd7j7ikzawG+B7wM6ALe6e7bw/v5BHAJkAE+5O53VPmpiIg0HBUqS80YKKPUJ/oa/6x27ZWGZWargQ8B6939ZCBOsHfLvxDs7XI8cJAgoCf8ftDdXwR8JTxPe76IiEREgb/UjFypT9tENf7ZbDWnNE4668RV5iONLQG0mVkCaCfo7PZG4Mbw+LXAW8PLF4bXCY+fbUGdnPZ8ERGJgAJ/qRm5Up+JNvCKOO4nm3XicQX+0pjc/XngS8BOgoD/MPAIcMjd0+Fphfu35Pd2CY8fBpaiPV9ERCKhwF9qRi7jX6rUJ1EjGX+V+kijCjdrvBA4BjgS6ADOK3Jqriav1N4uZe35ov1eREQqS4G/1IxcO89SpT6xmqjxdy3ulUb2JuA5d+9092HgR8ArgUVh6Q+M3r8lv7dLeHwh0E2Ze75ovxcRkcpS4C81I7e4t1SpT61s4KUaf2lgO4GzzKw9rNU/m6B9893A28NzLgZuCS/fGl4nPP5Ld3e054uISCQmbecpUi0DqTQtiVjJjHqwuLcGAn/V+EuDcvcHzexGgpadaeAx4GrgNuB6M/uncCzX8vk7wH+Z2VaCTP9F4f1ozxcRkQgo8JeaMZDKlNy8CyBu0Wf8gxp/fVAmjcvdLyfYyLHQNop05XH3IeAdJe5He76IiFSZIhipGf2pdMkyH4B4PPrAP5N1VOIvIiIi9UiBv9SMvqE081pKB/61UuOvjL+IiIjUI0UwUjO6+1Msnddc8njcoq/xT6urj4iIiNQpBf5SM7r6Uyyd11LyeDzMtGcjDP4z2awCfxEREalLCvylZhzoS7K0o3TGPxF204ky659xFPiLiIhIXVLgLzUhmc7QO5SeMPCPhf3zo6zzz2Sz2rlXRERE6pICf6kJB/uHASYs9ckF3BmPLvBPZ1TjLyIiIvVJgb/UhAN9SYCJF/fmAv9MdIF/1hX4i0h1DaTSUU9BROYIBf5SE7r6UwATlvrkAu50NluVORWjrj4iUm2f/NGTUU9BROYIBf5SE7r7cxn/ibr6RF/qE/TxV+AvItWzo3sg6imIyByhwF9qQldfmPGfoNQnX+Mf6eJeZfxFRESkPinwl5pwoC9FU9yYP8HOvbFcqU+ENf4K/EVERKReKfCXmtDdn2RpRwtmpYPqXMY/G2VXn6yTiOm/jYiIiNQfRTBSE7r6UhOW+UDh4t5oM/4xZfxFRESkDinwl5pwoD814cJeKFjcG3Hgr8W9IiIiUo8U+EtN6OpLTtjKE7S4V0RERGQmJg38zewaM9tvZk8VjP2rmf3OzDaa2c1mtigcX2dmg2b2ePj1zdmcvMwd3f2pSQP/eFhbH2Xgn85miU+wDkFEpNL0iiMilVJOxv+7wLljxu4ETnb3U4HfA58oOPasu58Wfr2/MtOUuWwglWYglSmj1Cf4Hm2NP8Tj+jMsIiIi9WfSwN/d7wW6x4z93N1ze4g/AKyZhblJg8j38K+DjH8mm1WNv4iIiNSlStT4/xXw04Lrx5jZY2b2KzN7TakbmdmlZrbBzDZ0dnZWYBpSr7r7J9+8C2qjxj+tGn8RERGpUzMK/M3sU0Aa+H44tBdY6+6nA38P/MDMFhS7rbtf7e7r3X398uXLZzINqXNd/UmASUt9YpZr55md9TmVks26avxFpKqiS3WIyFwz7cDfzC4GLgDe7R7sqOTuSXfvCi8/AjwL/EElJipz14EyS30SYW19hHF/kPFXjb+IiIjUoWkF/mZ2LvBx4E/cfaBgfLmZxcPLxwLHA9sqMVGZu8ot9RnZwCu6yF99/EVERKReJSY7wcyuA14PLDOz3cDlBF18WoA7LSh7eCDs4PNa4LNmlgYywPvdvbvoHYuEuvqStDXFaW+e+NcxV2ITeY2/Sn1EpIr0iiMilTJp4O/u7yoy/J0S594E3DTTSUlj6epLTZrth+h37s2Gj5vrLiQiIiJSTxTBSOQOlLF5F4zU+EcV+Of2D0ioxl9ERETqkAJ/qbgHtnVx1+Z9ZZ/f3Z+ctKMPjLTzjGoDr9wbjphKfURERKQOTVrqIzJVV93zLPt6hjj7hJVlnd/Vl+KEI4p2fR0lFnGNf8bDjL8W94qIiEgdUsZfKm5wOMNAKlPWue5OV1+KJWXU+Cci3rk3k8nV+CvwFxERkfqjwF8qLjmFwL8vmSaVybKsY/JSn3jkNf5BG1HV+IuIiEg9UuAvFRdk/NNlndvVV14Pfxhp5xlZjb+rxl9Eqs/0miMiFaLAXypuaDjLQCqTb385ka7+JEBZi3vz7Tw92sW9qvEXERGReqTAXypuaDgo8xlKT17uk8/4l9POMxf4Z6LZuTetGn8RERGpYwr8peIGw8C/P1lG4N9ffqlPLOJ2nllX4C8i1ecRfcopInOPAn+puORwkJEfLGOBb1dfUOqzZAoZ/2xEfwTTWQX+IiIiUr8U+EtFZbJOKizF6S9jge+BvhTzWxK0JOKTnhuvkQ28cm1FRRqRmS0ysxvN7HdmttnM/tDMlpjZnWa2Jfy+ODzXzOxKM9tqZhvN7IyC+7k4PH+LmV0c3TMSEWkcimCkonL1/UBZLT27+1NllflAweLeTEQZ/3yNfyQPL1Irvgb8zN1fArwU2AxcBtzl7scDd4XXAc4Djg+/LgWuAjCzJcDlwJnAK4DLc28WRERk9iiEkYoaHfhPnvHv6k+W1dEHRtp5RtXVZ6TGX/9tpDGZ2QLgtcB3ANw95e6HgAuBa8PTrgXeGl6+EPieBx4AFpnZKuAc4E5373b3g8CdwLlVfCp1Re08RaRSFMFIRQ1OMePf1Zcqq6MPBIt7YxblBl5q5ykN71igE/hPM3vMzL5tZh3ASnffCxB+XxGevxrYVXD73eFYqfFRzOxSM9tgZhs6Ozsr/2xERBqMAn+pqKHhkVab5WX8yy/1gaDcJ7oa/+C5xRT4S+NKAGcAV7n76UA/I2U9xRT7z+ITjI8ecL/a3de7+/rly5dPZ74iIlJAgb9UVGGpz2TtPLNZD2r8O8or9YEg8C9nY7DZkNs+QBl/aWC7gd3u/mB4/UaCNwL7whIewu/7C84/quD2a4A9E4yLiMgsUuAvFVUY+E/WzvPw4DCZrE8p45+IxSLL+KfDjL/aeUqjcvcXgF1m9uJw6GzgaeBWINeZ52LglvDyrcB7w+4+ZwGHw1KgO4A3m9nicFHvm8MxERGZRYmoJyBzS2Gpz2TtPHObd5XTwz8nyhr/jGr8RQD+X+D7ZtYMbAPeR5BEusHMLgF2Au8Iz70dOB/YCgyE5+Lu3Wb2OeDh8LzPunt39Z6CiEhjUuAvFTU4hYz/gXDzruVldvUBSMRjkQf+qvGXRubujwPrixw6u8i5DnygxP1cA1xT2dmJiMhEVOojFTWqxn+yjH9fkPEvt50nRL24Vxl/Eak+veKISKUo8JeKygX+ZpO388xl/KfU1ccs312n2nJvOFTjLyLVFE2qQ0TmIgX+UlG5wH9xezMDk3T16epLErPg3HLFY5bvrlNtGQX+IiIiUscU+EtF5Rb3LulonrTU50B/iiUdzVMKpBPx6DL+KvURERGRelZW4G9m15jZfjN7qmBsiZndaWZbwu+Lw3EzsyvNbKuZbTSzM2Zr8lJ7chn/JR3Nky/u7U2ybAr1/RCU+kRd4x+P6f2yiIiI1J9yI5jvAueOGbsMuMvdjwfuYmT3xvOA48OvS4GrZj5NqReDwxkSMWNBaxP9kwT+U921F8INvDyqPv5h4G/K+IuIiEj9KSvwd/d7gbE9li8Erg0vXwu8tWD8ex54AFiU29FR5r6h4SytTXE6WuIMTlbq05ec0q69EHb1yUQT+Od2DI7HFfiLSPXs6h6IegoiMkfMpGZhZbiyL3daAAAgAElEQVQDI+H3FeH4amBXwXm7wzFpAIPDGVqb4rQ3xyfP+Pelpl7qE7PI+vinVeMvIhGYrGxSRKRcs1GsXCwqGhepmdmlZrbBzDZ0dnbOwjQkCsnhDK1NMdqbExP+sRoaztCXTE+51CcRMzIRlfrkFhWrq4+IiIjUo5kE/vtyJTzh9/3h+G7gqILz1gB7xt7Y3a929/Xuvn758uUzmIbUkqF0hrZ8xj+NlwjSp7NrL0Sb8c+oxl9ERETq2EwC/1uBi8PLFwO3FIy/N+zucxZwOFcSJHPfYCpX6pPAfaS951gju/ZOfXFvVDX+adX4i4iISB1LlHOSmV0HvB5YZma7gcuBLwA3mNklwE7gHeHptwPnA1uBAeB9FZ6z1LBgcW+M9uY4AAOpNG3h5UIju/ZOI+MfWamPavxFRESkfpUV+Lv7u0ocOrvIuQ58YCaTkvo1OJxhfmuiIPDPsLTIebmM/7Ip1/jHGByOZqFb7g1HTKU+IiIiUoe0E5FU1FDY1aejJXhPOVBigW9nmPGfalefWCzCDbwyyviLiIhI/VLgLxWVTGdpa4rny3v6S/Ty7+pLMa8lQWvT+DKgiSRilu+nX235Gn8F/iIiIlKHFPhLRQWLe2N0NCfy14vp6k9OeWEvhIt7I+zqEzMwlfqISBVF84onInORAn+pqKH0yAZeAP3J4hn/YNfeqQf+iZjl++lXW8adREz/ZURERKQ+KYqRihoaHunjD6Vr/Kezay9EXOOfdZX5iIiISN1S4C8V4+4MDWdpCfv4Q+nA/0BfcsqtPCHiGv+MAn8RERGpXwr8pWKS6aAEp7UpRnvLSB//sTJZp7s/xfI6q/HPugJ/Eam+vhIlkyIiU6XAXyomt5C3rSlOe1PpUp9DAymyPvXNuwDiZvmNtKotnc2qlaeIiIjULQX+UjFD6SDIb22Kk4jHaE7EirbzPBBu3jWdrj6JeHSBv2r8RUREpJ4p8JeKGRoOSn3awmx/R3O8aDvPrmlu3gVBqY8CfxEREZGpU+AvFZML8lubgl+r9uYE/cnxgf/Irr3TqPG36Gr80wr8RUREpI4p8JeKyZX6tIQZ//bmeNHFvV1hqc/0Mv6xyLr6ZLKuGn8RERGpWwr8pWKGChb3ArS3JIou7u3qT5KIGQtam6b8GIl4tBn/mAJ/ERERqVMK/KViChf3ArQ3Fc/4H+hNsaSjeVpBdCzCrj5ZZfxFRESkjinwl4oZt7i3JV4y4z+dMh8INvDKeJQ1/vovIyIiIvVJUYxUzNjFvW3NxUt9OvtS02rlCSNdfTyC4D/o6lP1hxURERGpCIUxUjFjS306Si7uTbJ8mhn/XFedKMp9Msr4i4iISB1TFCMVkyv1yQX+bc1xBoq08+yaYcYfiKTcR119REREpJ4p8JeKGRoeXerT0ZxgYDgzqiynP5lmcDjD0hnU+EM0Gf90NkvcFPiLiIhIfVLgLxUzNJwhZtAcFsK3t8TJZJ1kOps/ZyY9/GEk4x9FS89sFm3gJSIiInVLgb9UzGAqQ2tTHAuz4u1hyU/hAt/crr0zLfWJYhOvdDZLIq7AX0REROqTAn+pmKF0Jl/fD8EGXsCoBb5dYeA/3cW9iQgz/sHiXgX+ImYWN7PHzOwn4fVjzOxBM9tiZj80s+ZwvCW8vjU8vq7gPj4Rjj9jZudE80xERBqLAn+pmKHhbL6HP0B78/iM/4Gw1Ge6Gf9YpDX+rhp/kcCHgc0F1/8F+Iq7Hw8cBC4Jxy8BDrr7i4CvhOdhZicCFwEnAecC3zCzOCIiMqumHfib2YvN7PGCrx4z+4iZXWFmzxeMn1/JCUvtGhzO0NI08ivV0ZzL+I8E/rmM/5KO6QX+US7uVcZfBMxsDfAW4NvhdQPeCNwYnnIt8Nbw8oXhdcLjZ4fnXwhc7+5Jd38O2Aq8ojrPQESkcSWme0N3fwY4DYKPfYHngZuB9xFkfr5UkRlK3UgOZ2hNFMn4JwtKffpTLGhN0JKYXnIv10c/qsBfNf4ifBX4GDA/vL4UOOTuuf/ou4HV4eXVwC4Ad0+b2eHw/NXAAwX3WXgbERGZJZUq9TkbeNbdd1To/qQODQ5naGsuDPzHZ/w7e5PT7ugD5HfOjarGP6ZSH2lgZnYBsN/dHykcLnKqT3JsotsUPt6lZrbBzDZ0dnZOeb4iIjJapQL/i4DrCq5/0Mw2mtk1Zra4Qo8hVfbTJ/dy28a9ZZ8/NJzN9/CHoJ0nQH/B4t7nDvRz1JL2ac8p0oy/awMvaXivAv7EzLYD1xOU+HwVWGRmuU+Q1wB7wsu7gaMAwuMLge7C8SK3yXP3q919vbuvX758eeWfjYhIg5lx4B92b/gT4L/DoauA4wjKgPYC/1bidsrk1Lhv3/8cV9/7bNnnDw1nJlzcm8062w708aIV86Y9p0g38Mp4/o2HSCNy90+4+xp3X0eQ8Pmlu78buBt4e3jaxcAt4eVbw+uEx3/pwY5+twIXhV1/jgGOBx6q0tMQEWlYlYhizgMedfd9AO6+z90z7p4FvkWJBVvK5NS+/mSanqH05CeGgsW9pUt9nj80yNBwdkaBf67UJp3NTnJm5QWLe6v+sCL14OPA35vZVoIa/u+E498Blobjfw9cBuDum4AbgKeBnwEfcPfMuHsVEZGKmvbi3gLvoqDMx8xWuXuuPuRtwFMVeAyJwEAqQ3+y/MA/OZydcHHv1s4+gIpk/COI+8m4Mv4iOe5+D3BPeHkbRZI87j4EvKPE7T8PfH72ZigiImPNKPA3s3bgj4C/Lhj+opmdRrBQa/uYY1JHgoz/MO6e3413IkPDGdqaRwLjpniM5niMgeEgkffs/jDwXz79wD8ejzbjrxp/ERERqVczCvzdfYDgY93CsffMaEZSM/pTaYYzHmzM1Tx5+83BMe08IVjgm8/47+9jSUczi6fZwx/Ib6AVTY1/Vn38RUREpG6pbkGKSmeyDA0HWfWeoeFJz3f3MOM/JvBviudr/Lfu75tRth+iXdybdRT4i4iISN1S4C9F5cpzAHoGJw/8U5ksWYfWprEZ/wQDqQzuztbOPo6bQX0/jATekWT8s1mV+oiIiEjdUuAvRQ0kRwL/w2UE/rlPB1oSo3+l2pvj9KfSdPWnODQwPKOFvTAS+Ee1gZcy/iIiIlKvFPhLUX0F3XzKKfVJhp8QjCv1aQ5Kfbbun3lHHyjI+HsUGX8F/iIiIlK/FPhLUQMFu+32DE7e0nMwDPzHLe5tTjCQSlcs8E/kdu7NVDfwz2YdV42/iIiI1DEF/lJUf0GpTzkZ/1ypz0QZ//bmOEcubJ3RvKIq9cl9wqAafxEREalXCvylqMKNu8pZ3JvP+DeN/pXqaE4wkMzwbGcfxy2fV9Z+ABOJanFv7vFiCvxFRESkTinwl6L6C0t9hiYv9RkqUerT1hzPl/rMtMwHoqvxzwX+yviLiIhIvZrRBl5S/zbtOUw8ZrzkiAWjxnOlPjErL+OfD/zHlPp0tMTpTabpGUpz3PKOGc93pI9/dXfuzZUWxWN6rywiIiL1SVFMg/vw9Y/zuZ88PW48t7h3+fyWMtt5ll7cm0vOVzLjn67y4t5cxj+uhL+IiIjUKQX+DayzN8nW/X0c7B8f2Ocy/kcsbJvS4t6xNf7tBZ8AVDLwz0ZU6hOP67+MiNS+w4PDnPOVe9myrzfqqYhIDVEU08AefK4LKN61pz+VprUpxuL2prLaeQ6V6OPf0RxUkyVixtFLK1fqU/WuPqrxF5E6cs8z+3lmXy9X/nJr1FMRkRqiwL+BPbitG4DeIot3+5Np5rUkWNjWVFbGv1Qf/9wbgaOXttNUgWx5LKKuPulwTYH6+IuIiEi9UuDfwHIZ/96hYXxM6Ux/Mk17c4IFrU1lLu4t3se/oyW4XokyHyhc3BtVjb8CfxEREalPCvwbVHd/it/v62NJRzNZh/5UZtTx/lSG9uY4C9oS9Aylx70xGCuX8W9JjP6VamsKSn0qFfhH3cc/odW9IiIiUqcU+Deoh8Js/xtfsgIIsv6FBlJBqc+C1iYyWR/3xmCs5HCGlkRs3AZd81uDwP/4FfMrMu/Idu7Nt/NU4C8i9WOypI2INBYF/g3qgW3dtDbFePWLlgGMW8Dbl8zQ3pJgQVtTeHzicp+h4cy4Mh+AE1ct4F/+7BTOPfmIisw7qox/WqU+IlJHZrpLuojMTQr8G9SDz3XzsqMXs6SjGSiS8U+mmdcSZ0FrGPhPssB3cDgzbmEvBItx3/nytbQ2jT82HYlwA63IavyV8RcREZE6pcC/AR0eGOZ3L/Rw5jFLRzL6YwL73OLehfmM/8QtPYeGs0Uz/pWWi7sja+epGn8RERGpUwr8G9BD27txhzOPWZKvwR/b0rM/laEjXNwLk5f6DIY1/rPNzIjHjGxEpT4xfXwuInVAtf0iUkwi6glI9T24rYvmRIyXHrUoH/CPDewHUmk6wsW9MHmpz9BwpmLlPJOJx6zqGf/cTsG5UiMRERGReqPAvwE98FwXpx+1aFSg3lOQ8U+mMwxnPAj8w1Kfw5Nk/JPDWdqqFfibkQk31KqWdEY1/iJSP7S4V0SKUfqywfQMDfP0nh7OPHYpAK1NcZrjsVEZ/YFk0LqzozmeLwWarMZ/cDhDa1N1fp0SMSNT3bhfNf4iIiJS9xT4N5iNuw6TdXjFuiX5sQVtiVE1/n3J4HJ7S4KmeIz25nhZpT7VWNwLEI9XP+OfcdX4i4iISH2bcamPmW0HeoEMkHb39Wa2BPghsA7YDvy5ux+c6WPJzO05PAjA2iXt+bH5rU2jAv+BVC7jH/x6LGhtmryPf7p4O8/ZELfq1/jn3mgkVOojInVES3xFpFClMv5vcPfT3H19eP0y4C53Px64K7wuNaCzNwnAigUt+bEFrYlRgX1/KngT0NESBPIL25om7+OfytJSxcW92Sp3rFCNv4jUE71SiUgxs1XqcyFwbXj5WuCts/Q4MkX7eoZY2NY0amFvkPEvCPyTucA/zPi3JSat8U8OZ6q2uDcRs3wgXi3awEtERETqXSUCfwd+bmaPmNml4dhKd98LEH5fMfZGZnapmW0wsw2dnZ0VmIaUY1/PECvmt4waW9CWGNXVpz9ZpNSnnJ17q7S4Nxaz6u/cm2/nqcBfRERE6lMlIrVXufsZwHnAB8zsteXcyN2vdvf17r5++fLlFZiGlGNfT5KVC1pHjc1vGZ3xHxhT6rOgrWnCdp7pTJZ01qvWxz8Rs3wgXi3K+ItINR0aSM3o9qrtF5FiZhz4u/ue8Pt+4GbgFcA+M1sFEH7fP9PHkcro7E2Oqu+H8aU8uVKf9nzGPzHh4t6hdLDwtWp9/CPYwEs1/iJSTb99tqui93f1vc+y7rLbSKYzFb1fEakvMwr8zazDzObnLgNvBp4CbgUuDk+7GLhlJo8jlZHNOvt7h8Zn/FubGBzOMBw2x+8Pu/rMy9f4N9GbTJMtEWwPhudXq9QnHjMy1a7xdwX+IlI/xr5SfeOeZ4GRfVpEpDHNtJ3nSuDmcIfABPADd/+ZmT0M3GBmlwA7gXfM8HGkAg4OpBjO+Lga/9wmXb1DaZZ0NNOfTBOzkUB+YVsT7tCXSrOgtWnc/Q4N5wL/amX8Y5GV+iRi2vpCRERE6tOMAn933wa8tMh4F3D2TO5bKm9/2MpzbMY/F8z3Dg2HgX+GjuZEfsv33PGeweGaCPwTESzuzZUWKe4XkXqm2n+RxqYwpoHs6xkCYOWC4hn/XJ3/QCpNe8tIEL+gbfTxsYaGgxKhagX+sQhq/LPK+ItgZkeZ2d1mttnMNpnZh8PxJWZ2p5ltCb8vDsfNzK40s61mttHMzii4r4vD87eY2cWlHrNRVXqTcBUpiggo8G8o+3vCzbvmj8n4t41k/AH6kul8K08YyfiX6uzzQviGYvmYEqLZkohZyfUGsyWtrj4iAGngo+5+AnAWQSe3Eym9aeN5wPHh16XAVRC8UQAuB84kaAhxee7NgkzPnU/vY91lt+U3acxTil9ECijwbyD7SgTo+Yx/GPgPpDL5zbtg5I1BqV7+O7r6ATh6SXtlJ1xC0NUnW5XHysmEj6c+/tLI3H2vuz8aXu4FNgOrKb1p44XA9zzwALAo7PR2DnCnu3e7+0HgTuDcKj6VOefa32wH4Hcv9AClPzHwKq+PEpHaosC/gezvTbKovWlcSU6+hj/cxKsvmaa9OT7+eImM/67uAea3JljUPr7+fzbELboaf2X8RQJmtg44HXiQ0ps2rgZ2FdxsdzhWalymKbf/Su4NgOJ7ESlGgX8D2dczxMoxZT5QuLh3pMZ/XkHGf2Hb6DcGY+3oHuDope35xcCzLRGvfuCfVeAvkmdm84CbgI+4e89EpxYZ8wnGxz6OdngvU1d/sOHXLzaP2TZnzE+6Wq/TIlKbFPg3kH1FNu8CmJdf3BuW+iQztBcE/mOPj7Wza4C1VSrzgbCPf1QZf/3RlAZnZk0EQf/33f1H4XCpTRt3A0cV3HwNsGeC8VG0w3v5Sr4yKfMvIgUaJvB/ZMdBuvtntgV6vdvfMzRuYS8EgfS8lkQ+4x8s7o2POj6/JVG0xj+TdXYdHGDtko7Zm/gYiQi6+mSyjlnQUUikUVmQLv4OsNndv1xwqNSmjbcC7w27+5wFHA5Lge4A3mxmi8NFvW8OxyRvaq81YzP5qvEXkWIaIvBPZ7K861sP8K37tkU9lchks05nb3JcK8+c+a2Jkot7IVjgW6yrz97DgwxnnKOXVi/jH4ugxj+TdS3sFYFXAe8B3mhmj4df5wNfAP7IzLYAfxReB7gd2AZsBb4F/C2Au3cDnwMeDr8+G47JLFGJj4jAzHfurQv7e5Ok0ll2HxyMeiqR6R5Ikc76uM27cha0NtE7NIy7058anfGH8I1BkT7+O7sHAKpa6hNFjX8m66rvl4bn7vdTOhU9btNGD9LLHyhxX9cA11Rudo3rG/ds5bkD/VFPQ0TqQENk/PceDgL+fYeHIp5JdEpt3pWTC+wHhzO4UzTjX6zUZ2dX9QP/eCwWSY2/6vtFpFqm8nLzxZ89U/a5KvQRaWwNEfjvORQEvft6Gzfw3x9u6rK8SI0/BIF9b3KY/mQGYNTiXgg+ESi2uHdH9wBNcePIRW0VnnFpcSOSGn9l/EWkWh7beWjU9aeeP1z2bddddhsHwtd8D0N9vXqJCDRI4J/L+L9weKhhFzbtLzPj358MynnGlvosbGvKL/4ttLNrgDWL26saFEeR8c9knUS8If67iEgNuOeZ0W05SzWnyG2gONYVP34agNuffAFQpl9EAg0RyeQy/sl0tugC1UawryeX8S8e+Odq/PvDTWDGl/okimb8d3YPcFQVy3wg6OoTSamPMv4iUkMODaR43b/eM+l5yXRm9icjInWhIQL/XMYfRgLgRrOvZ4glHc20JOJFjwddfdL5Up+O5vGlPr3J9LiAe0dXP0dXOfCPRdDOM6safxGJULFXvL5k8U0Vi9Grl4hAwwT+Q/mdaF/oacw6//29SVaUyPZDUOOfyToH+oI3Rh0t8XHHAXoLFvgeGkjRM5SuaitPCDL+2SqXbCnjLyLV9LsXeit2Xw1a4SoiRTRM4H/qmoVA43b22d8zxIoSrTwhyPhDsA4CipT65HfvHckw7Qg7+lS71CceM9KZbFUfM5PNkogr8BeR2lFub/43fOme/GW9CRBpbHM+8E+lsxzoS3LaUYuAxs347+tJsnKCjP/81iCjn/v5tDcXz/gXtvTM9fCvdsY/HlWNv0p9RCQixRpTlPuKtPfw0JTag4rI3DXnA/99PUO4w7qlHSxub8r3s28kmazT2ZcsuXkXjGT094YZ/3ljMv6LwsC/8OcXxeZdEC7urXLaKusq9REREZH6NucD/z2HgoW9qxa1snJBa0MG/t39KTJZZ0WJVp5QkPEPF0K3j1nce+qaRcxvTfCTjXvzYzu6+lk2r2XcubMtkox/RoG/iERHFToiUglzPvDPZbBXLWzjiIWtDVnqk3uzs6LE5l0AC9tGMv7N8RjNidG/Gm3NcS487Uhuf3JvviXqjq6Bqpf5QFjjrw28RKSB7Ao/YRURmYk5H/jvCTPYRy5qZeX81oZs57m/d+LNu2Ak47+vZ4j2luItP9+5fi3JdJZbH38eCP4QVbuVJwSBv3vQYrNaMu4kFPiLSEQ+c8smHt15cNSY6vZFZKrmfOC/99AQC9uaaG9OsHJhKwf6kgxXuSNM1PaHb3YmrvEPAv/hjI/r4Z9z8uoFnLBqAT/csItkOsPeniHWRpDxzwXg1azzV8ZfRKK2rXP0Lr02he78B/qCnX//7wM7RrVlFpHGMvcD/8ODrFoYBLxHLGjFHTp7Gyvrn/uUY9m80hn/1qZYPqAe28M/x8x45/o1PPV8D3ds2od79Rf2QrCBF1DVOn/V+ItI1Ip19pmqr921hc/csqkCsxGRejTtwN/MjjKzu81ss5ltMrMPh+NXmNnzZvZ4+HV+5aY7dXsODXHkojZgpNSl0er89/UOsbSjeVzdfiEzy7fsnGix7ltPX01zIsZX7vw9UP1WnlCQ8a9yqY8CfxGZC7r7U3zr3m2su+y2qu+JIiLRmknGPw181N1PAM4CPmBmJ4bHvuLup4Vft894ljNQmPHPlbrsb7DAf7LNu3Jym3iNbeVZaFF7M+ecdATPHQg+cl67pKMyk5yCeCz4ta3mAt9M1knE5vwHZCJSwyq1Y/n+3iSfv30zACkF/iINZdqRjLvvdfdHw8u9wGZgdaUmVgmDqQwHB4bzGf8jwjcALzTQ7r3uzjP7elm9qPzAf+zmXWO9c/1R+fOWzWue+SSnKLeBblVLfVTjLyIRGxv3T3dx7+a9PSXvU0TmtoqkMM1sHXA68GA49EEz22hm15jZ4ko8xnTsDTv65DL+S9qbaYobLzRQZ58Hn+tmV/cg55+yatJzcwt8OybI+AO88rilrFncxtol7WVvGV9J8Xjwa1vVUp9sVoG/iERq7Cver57pjGQeIlK/Zhz4m9k84CbgI+7eA1wFHAecBuwF/q3E7S41sw1mtqGzc3ZevAp7+EOwKHTF/NZZKfV57kA/l9/yVM11DPrhw7uY35LgvJMnD/xzGf9Si3tzYjHjG+8+g8+/7ZSKzHGqIqnxz6LAX0QiNTY7/9ttXdFMRETq1owCfzNrIgj6v+/uPwJw933unnH3LPAt4BXFbuvuV7v7endfv3z58plMo6T8rr0LR8pcVi5omZXFvdc/tJNrf7uDDdsPTn5ylRweHOb2J/fyJ6cdSdsk5TtQkPEvYyfeU9cs4mVHR/NhTjz8lCGdreybrPu2dJZsc5fJZtXHX0QiVaka/0Kq9BFpLDPp6mPAd4DN7v7lgvHC1PLbgKemP72ZyWX8jygI/Gdr994HwszLfVtq56PXW5/YQzKd5aKXry3r/PlllvpELZd5r2Tcv2VfL+/5zkP856+3Fz2eznq+jaiISBTGtvN8ZMfME00Pb++e8Pi6y27j337+zIwfR0Rqw0wy/q8C3gO8cUzrzi+a2ZNmthF4A/B3lZjodOw9PMjSjmZam0ay3Svmt7Kvwot7+5JpntoTLJa6f+uBit73TNzw8C5OWLWAk1cvKOv8BW3lLe6NWiJe+Yz/jzfuBWBDiT+k2ax27hWRaBVWNw5nsuzsHpjxfb7vPx9m3WW3jRr7j7u38rOn9uav//svt874cUSkNkw7tevu90PRbQMjbd9ZaO/hIVaN6WZzxMJW+lMZ+pLpCdtWTsWG7d1kss4r1i3h4R3dHOxPsbij+t1uCm3ac5gnnz/MFX98YtkLcHMZ/0r9XGZLzCpb4+/u/GTjHgAe23GQbJHsvrr6iEjUDg6k8pfvfHrfrD3Ov94RZPi3f+Ets/YYc9mD27o4clEbR0WwwaXIZOZ0Y/K9h4byC3tzjlgwtZaeP35iD7/f1zvhOQ8+100iZnz4TcfjDr9+Nvqs/w0P76I5EeOtp5ffYTXfzrPGA//84t4K1btu3tvLts5+Xnb0YnqTabbs7xt3Tibr+bUFIiJR+OovtuQvV7rc/wEtFK6Yd179AK/54t1RT0OkqLoO/A8NpPj1BKU1ew4PcuTC0Rn/FeHuvfvKqPPv7E3y4esf40PXPTZhdvnBbV2cumYhZx6zhPmtCe77fbSB/9Bwhpsfe55zTjqCRe3lf/Iwsri3tkt9cpn3dKYyf/l+snEP8ZjxyfNPAODRnePLfTJZz5cYiYhEZcP2bn765N5p9/Av5aKrH6jsHYpITarrwP8/7t7KX/7nQ0U7sfQl0/QOpVm1qHjGv5zA/2dP7SXr8LsXern5seeLnjOQSrNx92HOPHYpiXiMVx63lPu3Hhi3CKuavvfb7fQMpbno5UdN6Xarw5/VyjJ2+Y1SvILtPIMyn7288rilnLF2EUs7mosumMuo1EdEasDbv/lb/ub7j9I3lK74fWcneE1190j/rolIZdR14H/2CSsZzjj3bRmfYd9bpJUnFOzeW0bg/+ONe3nRinmcsnohX/75MwwNZ8ad88iOg6SzzpnHLAHgNccv5/lDg2w70D/l51MJW/f38qWf/543nbCSVx63dEq3PWXNQu75h9dz8uqFszS7yohXsNTnyecPs7N7gAtOXYWZcfraxTxaJPBPZ51ErK7/u4hInZgoAM/52E0bK/645195H7tKLBg+5hO38+av3KvgX6TO1XUks/7oxSxsa+IXm8cvctoT1vAfOSbj396cYH5rYtLOPvt6hnh4ezcXnLqKT5z3EvYcHuLa32wfd96D27qJx4z163KB/zIA7i/yZmS2pTNZPnrDE3Q0x/nnPz15WrvqrlvWMQszq6xcAF6JjP9tG/eSiBnnnHQEAC87ejHbDvTT3Z8adV426/lFxSIis2lPuOt8tf3uhd4Ja9O37O/juod2VXFG9Wd7REk/kXLVdeCfiMd4w4uXc88zneOCwFIZfwhKWfb1JCe879uf3Is7XHDqkbzyRct43R8s57WmguQAABpvSURBVD/u3sqhgdEB4YPPdXHy6oX5TjhHL+1g7ZL2SPr5f/NXz/LE7sP801tPYcX82i7XmYlc4n2mNf65Mp9XH78svxYitynZ2Kx/WjX+IlIlhwaKbyRYbd+4Z3wbz6f2HC7rtpmsc8emF+r+E4KbHtlNX3LisqpnO/u48Ov38/NNL/D6L91TnYmJTFNdB/4QlPt096d4bMyCzEd2HKS1KVa0Xv2IBZNv4vWTjXt5yRHzedGKeQBcdt5L6E2m+cY9z+bPGRrO8MSuw5wVlvnkvOb4Zfz22S6GM5XdWXYim/Yc5mt3beGCU1fxllNXTX6DOpbL+M90F8vHdh3i+UODXHDqkfmxU9csJBEzHhnz+6QafxGplgv+/f6opwDAF382fuOusS+7m/f2kEyPL4P99n3b+Ov/eoTbntw77lhUvvqL37OpzDcuAI/tPMhH//sJPn3zkxOed/a//Yondh/mb77/6EynKDLr6j7wf92Ll5OIGb/YvD8/tr93iFse38PbX7aGpvj4pxhk/EsH/nsODfLIjoP88UtHAsITVi3gbaev5ru/3s4PH96Ju/PozoOkMlnOPHZ84N+fyvDYzkMVeIaTOzSQ4kPXPcbCtmY+d+HJVXnMKOW7+syw1Od/Hnue5niMPzpxZX6stSnOSasXjlvgm85m1c5TZA645fHnWXfZbTy5u/wAcLY8uK2Ljbur83diNuzrGeK8r93HZ/5n07hje8Ny2s7eiT9drxZ356u/2MKFX/910eO9Q8P8xbceGLXGYSAVvKHZP8FzuPmx3fnLpcpPhzPZuv/kQ+aOug/8F7Q2ceaxS7iroM7/e7/ZwXA2yyWvPrbobVYuaGF/b5IbNuwq+hHebeEurm85ZXTm/NNvOZH16xbz8Zue5MPXP85dm/cTM/L1/Tl/eNwyYgY/e+qFmT49IHjR+MXT+/jI9Y9x5V1bRn2SMJjK8FfffZhdBwf5+l+cHvnGYdWQ7+M/g517Dw2kuPGR3Vzw0lUsbGsadexlaxfzxK5D+Z+zu5N1lPEXmQM+fP3jAPzx16PPqr/z6gf4kxKBaO0KAtjhTDa/FmrDju5xZ303XBNXyXj38OBw0SYbpfx80wusu+w2njvQn59HqYTRT596gd8828XX7tpS9PhYx3ziNtZddht/98MnSp6zq3sAd+f4T/2U//3jp1l32W186Y7xn6LMVFdfkk17DpesMvjSHc/wj//9xLif3W+2Hii6p9END+/i1CvumPG8Lvnuw3zuJ0/P+H7msmpWhuTU9k5NZTr7JSv57E+eZkdXP8vnt/BfD+zgzSeu5JgSC1XPOekIbn9yLx+7cSOfueUpzj3pCP7sZWt45XHLiMeMn2zcw8mrF4xb6Lqko5n/uuRMrrpnK1++8/dkHU5evSDf/z5nYVsTbz1tNdf8+jlOXbNwSpto5bg7m/b0cNOju7n18T109aeY35rgfx7fwz3P7OfKd53OEQta+eAPHuWxXYf4xl+cwVnHTq2LT70aaec5/fv4/oM7GUhl+H9eM/7N4cuOXsw1v36Op/f08NKjFuWzOAkF/iJzyv6eIVZMsX3xoYEUTfEYHVPY6HAwlaElERu3I3gxs7kjbyVc99AuXrRiPl+64xkGw0Dy2c7RC1r/p6D9dblxv7vnG1I89fxhXnLEfACyDs2JGCd+5mcMpDIcu7yDf37bKTy4rZsPv+n4Uffx9J4eMlnnlDULyWSdT978FBB0b1s7wS66z3b28bEbgy5J6UyWU6+4gwteeiTnnxwk/37zbBfPHehnMJWhvTnOLzbvK+sNzRW3buLq964HRt4Iff3urfzDOS8u86dS3C+e3sf/+t4GEjHjH855Mdfc/1z+U4mHPnU2B3pTnH/lffzXJa/g1S9axtfvDtZq7DkcVDP86h/fwMoFrfzFtx8cdb+nrF7I/9/enYdHWd0LHP/+Zs++EJAlhEVRwIVFRFBrEavgUrFWW7u53LZe9Vpve69V1Gsfay3V23pv9dZHa61rFdsKWipVS1GRB2RXQWQLAWIWSAJJyGSZ9dw/3jdpAgnJZHGYmd/neebJu83MWd6c98x5z3vOX39wXtuIUaPnL23bd+O5o7lzznjSPE5CkSi1TcEOzxJWHW6hMRhh9KD0tnxcvt3qiXHf5RO7jdM9r21hcKaXH110MntrGnE5hcK82GY+bgyEmfWr93j02smcc2JBTO/tTCRq8LeEyUm36nihSJRH/7GLi089gTMKc/v8+a9/WM4P//gR790xi4XrSsnL8HDzF0/s8+d2Jykq/l+aYFX8/7GtCrdTqG8OcdP5nbf2A0wamcu7d8xiU2kdizeV8dePK3j9owpOyPYy59ShfFxWz/xLxnf6XqdDuG32OM4eO4i7Fm3minbdgdpbcNXpVNQ3c8efPyY/w8P5Jw9u21fjD5DpdeFzHz1RVnVDgMWbyli0qYydB/x4nA4unDCEr04t5IunDObtrfu5e9EWLn10JVOK8lixs5oHrzyNS05P7n797Tn72OIfCEd4bvVevjCugAnDso/aP3WU9Q+9cV8tk0bmtrUQ9eSirZQ6fh3Z4lntD8Rc8Z/8wDJrvo/7LurR8aFIlAk/eYvrZ47ip3ZXzJJqf4eGqZtf3Mhts0/icHOI77+wIabwxENnrbgrdlZz+8IPqW8OHXVsmtvJN88uojkY4ed/+5Q7545nwdJtvLK+ZyMEzRib39btpqS6sW2ysbV7DnL1mYVcNbWQPTWNXPrYSgBmnTKYsQWZ1PityvDtC62JONuHdfLIXM5e8A9aQh2vI69/VAHAy2tLeXltadv2C3r50O5AdPF5Zb0VrnDU8NCb2zvsm/7z5ZxuD8n9nd+vY/eCS9v2rSq2Zmc+e8Fy9j502VGfu6W8nsffPfqBboBnV+3l2VV7Kcj0UOO37vR8e0YRX5kygq8+8UHbcXfNHc8tszpWXv++dT9vbz3AI1+b1Olnf+vpNW1h+9FFJ3d4QPpP/zqTcUMyWfJxhTW8emEOVYcDeJwOigZ1/GGwfX8D1Q0Bfvn2Dl67te8V/4fe3MbvVu5h8/0Xk+1zc9eizSzeVM5v3i2mZMGlPPDGp3xhXAEXTrC6C7+5pZJbXtrEh/dd1KH3xVuf7Gd3tZ9/u+CkDp//ht275O7FW/jAnjlbK/49VDQonZNPyOTvW/dTWd/ClKJczhyVf8z3iAhnjsrjzFF53Hf5RJZvq2LxpjJeWluKQ47u5nOks0bn885/zupyv8/t5KnrpvH1367h5j9s5NkbzqKivplFG8tZtbuGM0bk8PL3Z3RoNdpT08g1T66mxh9kalEuD155GpefMazD7LuXnzGcM0bk8oOFm6yC9sJxfHvGqJ4lVJJw9bGP/5KPKqhuCPDINZ0XQsNy0hie4+P9XdU4HcKiTVYfzrROfqgppRLH+UcMVXnrS5tY8eMLun1ffVOIkho/I/Ks4aEPNgY51Bgkv5OulXVNQSY/sIyHrjqdYCTK1WcWAvD8B/vwuBxU1LewdHMlP2nXCvrW1v28tbV/uobGy/XPrOty3z2vbeGJFcXMHDuIP20o4w9rSrs8tjNrSo7uSgRWS/zq3Qepbgh0aMR5b0c1q4q7HlL7WGHtbwPRs7+7obrbT2oa6w+PX3bTFam10g/whzWlR+Xlw29t5+G3Ov4YuenFjQDcMedksn1unA4hEIqS4XUSNf/8QQJQXtdxKNuv/fYDurJq/mxG5KZxuCVk34Hp39Ru7fZ9uDlEts/NXz+uaNv3yvrPeG71Xp5bvZdFt8xk2adVPLnCGvxlys+WseArp3PPa1t48MrT+K/XrTtPV0wazsj8dPYdbKQxEGkLb2ul//OSFBV/sEb3ecIecefuLlrru+JzO7nMHg2nxh9gf30LI49xW7Cnsn1unr/xLK56YjVft1soCvPS+Ob0IhauK+WWlzbx9HXT8LgcVDW0cN0za4kaWHr7eZw6vOtJtIoGpfPnm89ha0U9k0f2/XZTonEcY+beNSUHefit7YwpyODqqYXMGDuoQ0u9MYbfrSxh/NCstjkXOjN1VB5vbK7kvR3VTByWzX9dNoFvTC/q/8golcJEZC7wKOAEnjbGPDSQ33fkQ5r7DjbR0BIiw+M66o7e9v2HKcj0Eokazl6w/KjPuvyxlcw9bRhfGFdAeV0zV0wezv76Fu6zL/LzF1sjwfzkL/988PV3K/e0LT+QYn2fPzvUzGeHyro/sBd+cUSrN0Coj8M994fl26sYd++bne471Bhk6ZZKZo8fwogj5hvqTnf3nk0Xy/E28xfvdHvMuQ91f0yrO1/9mHNPKmgbfWrRLTMB+LC0jne3VzE4y0tlfQvbKw9zxeThlFQ3srvaz7dnjOq0x8WRWn9gtf52EoTWFL2n3UhPdy3aQnGVv8N7W/e3VvoBPth9kO37G+J+Vy9pKv5fmjCEJ97bTVF+OhfbkzH1RkGml4JMb7+Fa0i2j5e+dzYvry1l9vghnDU6H4dDOKMwh7sWbeGuRZu5/4pTueGZ9dQ0BFl404xjVvpbeVwOphTl9Vs4E0lri/+2ygbmnBrB53YSiRoeW76L/3tnF0OzfRQf8LN4UznDc3xcOWUEV00t5KQhmazYWc3OA34euWbSMVtNbpl1IicOzmTuaUM77Q6klOobEXECjwMXAWXAehFZYowZkBpxVyOunH7/3wHY+9BlBMNRdlf7GZThYe6vVx7z8yrqW3hm1R6eWWVV5ttf4JU6lmkPLsPrclJe18x9wNjBGZTYz0l8Y/pI7rj4FGY/sgJjDEtv/8JRDZHdTSbZfqjrrhr8PymP/6hWfbWq+GCHuwXt3fjc+g7rjyzb2bb84NJtbcuv3XoONf4g551UwM+WfsrCdaU8fNUZvLBmb9vdh/d2VDFvygiCXTxYeGSlvys9mW37s0NN/dLwfCxyPAwxNW3aNLNhQ99+AUWihhueXcfXpo3sMAzn8ezxd4v55ds7KMj0UtcU5OnrpzHrlCHxDtZxrzEQZs6v36estplsn4svTxpOcZWftXsOcdWUETxw5Wm4HMKyTw+waFMZ7++sJmqsZztaghHqmoOsvHM2HlfCD2qlUoSIbDTGTIt3OPqTiMwE7jfGzLHX7wYwxvyis+N7e53YXFZH6aEmbnv5w+4PVkolrJvOH8tT75fEOxh98ui1k5k3OfYBYWK5RiRNi7/TIbz43bPjHYyY3DrrRKobAjy3ei//+/VJWunvoQyvixU/voBVxTVtD0ILwq+umdTWnxbgy5OG8+VJw6lqaGHJRxW8urGMHQcauPfSCVrpVyr+RgDtn+4sAzoU4iJyE3ATQFFR77raPb96X9tzOkqp5JXolX7o+8SkPZE0Lf6JyhjDwcZgv3YvSjX+QBhjDFlHDKvamfK6ZoZl+3SEHpVQkrTF/xpgjjHme/b6d4DpxpgfdHZ8b68T5XXNNAbCuBxCQZa3bfjlstqmtgf2m0MRDjUGSfc4qWoI0ByMMKYgg2AkSiAUxeUUWgcRMxhy0zz4PA5KqhtxOYQsn5twNIrH6cBgXbxdDmFvTRM56W6C4SjGQDASIRiO4nQ42roeBSNR1u05yJiCTPLS3aR7XEwclk2WzxXTvCzhSLTDXCMigjHG/t4o/kCYnDQ3tU1Bq5tJbTNRYwhGomR6XYTsv5vL6gmGo+RnesBAps/Fjv0NOEQoq21i9KAMCvPSCEUNe6r95KZ7cDiENLcTl1NwOxzUN4fYuK+WEXlpRKOG/AwPoUgUh0NI9zjb4n7QH+SEbB+lh5oYlOkhw+PC63LQEAjhdlppVN0QwOUQstPcGGNNppib7qGsthmvy4HH5WBIlpdQxBCKRDnoD5Cd5iYSNdT4A6R7XAzJ8lLjD5LmceAPRDghy0tzKIJDhG2VhzllaBbbKhvsoSihJRSlMRhmRG4adU0h0txOnA4hP8PDvoON5KZ7yPC6CIQjeJwOwlHD4CwvNQ0BRIRAOEJzMEKax0ma20mN3zq3RKw4jx+WRSRqqKhrxuNyMDwnjV1Vfory0/G5nQTCVtiMAZdTCIajlNc14xQhas8rU1bbxElDMslJc1N6qAmnQ/A4HfjcTvbUNJLpdXGyPSTqnmo/I/PTqaxvIRyJEo4a3E4HUWMIRwx5GR6aQxEaA2FG5adTUd/CoAwPwUiUdI+TxkC4rU98SXUjpwzNoqKumUyvi5w0Nz63E4OhKRhhW+VhnCJMHJ6DPxCmoq4Zp0PwuZ2k20OBZvlcpHtcGANF+elstOeBcDgEn8tJhtdFYyBMWW0zk0bmcLjFmm+pJRhhzOAM6ptCHGqy0rS8tpkzR+XhcTlwOoTy2may09zU+AMMzvTiD4SpagjgdTkIhqOEIlHGDs6ksr6FA4et5zkj0SiRqPVgflmtlSfRqGFUQQYOsXqUGHsuH6/LQZbPCntVQwBjDI3BCNlpbrK8Lg41BsnwOgmEo7SEIgzNSWNoto/apiDNwQg+t4OWkJWf6R4nQ7J81PgD5Ka7yc/wMH1MPl5X7AOJxHKN0Iq/Ukod55K04v+5dPVRSqlkF8s1Qvs7KKWUiof1wDgRGSMiHuBaYEmcw6SUUkktafr4K6WUShzGmLCI3Aa8jTWc5zPGmK3dvE0ppVQfaMVfKaVUXBhj/gb8Ld7hUEqpVKFdfZRSSimllEoBWvFXSimllFIqBQxYxV9E5orIDhEpFpH5A/U9SimllFJKqe4NSMW/3VTslwATgW+IyMSB+C6llFJKKaVU9waqxX86UGyMKTHGBIFXgHkD9F1KKaWUUkqpbgxUxb+zqdhHDNB3KaWUUkoppboxUBV/6WRbhymCReQmEdkgIhuqq6sHKBhKKaWUUkopGLhx/MuAke3WC4GK9gcYY54CngIQkWoR2dfL7yoAanr53mSm6dI1TZvOabp07nhIl1Fx/v6427hxY41eJ3pM45vcNL7JrTfx7fE1Qowx3R8VIxFxATuBC4FyrKnZvzkQszKKyAZjzLT+/txEp+nSNU2bzmm6dE7TJfGlWh5qfJObxje5DXR8B6TFX6diV0oppZRS6vgyUF19dCp2pZRSSimljiPJMHPvU/EOwHFK06Vrmjad03TpnKZL4ku1PNT4JjeNb3Ib0PgOSB9/pZRSSiml1PElGVr8lVJKKaWUUt1I6Iq/iMwVkR0iUiwi8+MdnngRkZEi8q6IbBORrSLy7/b2fBFZJiK77L958Q5rPIiIU0Q+FJE37PUxIrLWTpc/iogn3mH8vIlIroi8KiLb7fNmpp4vICI/sv+HPhGRhSLi0/MlsSXDdSLWMl4sj9lx3iwiU9t91vX28btE5Pp4xaknelp2i4jXXi+2949u9xl329t3iMic+MSke7GUycmQv7GUtYmYvyLyjIhUicgn7bb1W36KyJkissV+z2Mi0tn8WZ0zxiTkC2u0oN3AWMADfAxMjHe44pQWw4Cp9nIW1lCqE4H/Bubb2+cDD8c7rHFKn/8AXgbesNf/BFxrLz8J3BLvMMYhTZ4Hvmcve4DcVD9fsGYX3wOktTtPbtDzJXFfyXKdiLWMBy4F3sSaTHMGsNbeng+U2H/z7OW8eMfvGPHuUdkN3Ao8aS9fC/zRXp5o57kXGGOfC854x6uLuPa4TE70/I21rE3E/AXOB6YCn7Tb1m/5CawDZtrveRO4pKdhS+QW/+lAsTGmxBgTBF4B5sU5THFhjKk0xmyylxuAbVj/WPOwChPsv1fGJ4TxIyKFwGXA0/a6ALOBV+1DUi5dRCQbq1D6PYAxJmiMqUPPF7BGOksTay6SdKCSFD9fElxSXCd6UcbPA14wljVArogMA+YAy4wxh4wxtcAyYO7nGJUei7Hsbp8OrwIX2sfPA14xxgSMMXuAYqxz4rjSizI54fOX2MrahMtfY8z7wKEjNvdLftr7so0xHxjrV8ALxHBdSuSK/wjgs3brZfa2lGbfApsCrAVOMMZUgnXhAIbEL2Rx82vgTiBqrw8C6owxYXs9Fc+bsUA18Kx9G/1pEckgxc8XY0w58CugFOsiVA9sRM+XRJZ014kelvFdxTuR0iOWsrstXvb+evv4RIlvrGVyQudvL8raRM/fVv2VnyPs5SO390giV/w768+U0kMUiUgmsAj4oTHmcLzDE28icjlQZYzZ2H5zJ4em2nnjwroF+YQxZgrQiHXbMaXZ/S3nYd0yHg5kAJd0cmiqnS+JLKn+32Mo47uKd0KkRy/K7oSOL7GXyQkd316UtQkd3x6INX59inciV/zLgJHt1guBijiFJe5ExI11QXjJGLPY3nzAviWE/bcqXuGLk3OBK0RkL9Yt/tlYrUi59u1FSM3zpgwoM8astddfxbropPr58iVgjzGm2hgTAhYD56DnSyJLmutEjGV8V/FOlPSItexui5e9Pwerm0WixDfWMjnR8zfWsjbR87dVf+Vnmb185PYeSeSK/3pgnP0UuAfrgY8lcQ5TXNh93X4PbDPG/E+7XUuA1qfArwf+8nmHLZ6MMXcbYwqNMaOxzo93jDHfAt4FrrYPS8V02Q98JiKn2JsuBD4lxc8XrNvOM0Qk3f6fak2XlD5fElxSXCd6UcYvAa6zRwuZAdTbXQveBi4WkTy71fVie9txpRdld/t0uNo+3tjbr7VHhRkDjMN6KPK40osyOaHzl9jL2oTO33b6JT/tfQ0iMsNOv+uI5brUl6eW4/3CehJ6J9aT3PfGOzxxTIfzsG7zbAY+sl+XYvWBWw7ssv/mxzuscUyjWfxzZIixWIVDMfBnwBvv8MUhPSYDG+xz5nWsEQNS/nwBfgpsBz4BXsQaLSLlz5dEfiXDdSLWMh6rK8Djdpy3ANPafda/2OdyMXBjvOPWg7h3W3YDPnu92N4/tt3777XTYQcxjHwSh3j2uExOhvyNpaxNxPwFFmI9vxDCaqH/bn/mJzDNTrvdwG+wJ+TtyUtn7lVKKaWUUioFJHJXH6WUUkoppVQPacVfKaWUUkqpFKAVf6WUUkoppVKAVvyVUkoppZRKAVrxV0oppZRSKgVoxV8ppZRSSqkUoBV/pZRSSimlUoBW/JVSSimllEoB/w+w8ucxMK2gFgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_frames = 10000\n",
    "batch_size = 32\n",
    "gamma      = 0.99\n",
    "\n",
    "losses = []\n",
    "all_rewards = []\n",
    "episode_reward = 0\n",
    "\n",
    "state = env.reset()\n",
    "for frame_idx in range(1, num_frames + 1):\n",
    "    epsilon = epsilon_by_frame(frame_idx)\n",
    "    action = model.act(state, epsilon)\n",
    "    if isinstance(action, torch.Tensor):\n",
    "        action = int(action.item())\n",
    "#     print(action, type(action))\n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    replay_buffer.push(state, action, reward, next_state, done)\n",
    "    \n",
    "    state = next_state\n",
    "    episode_reward += reward\n",
    "    \n",
    "    if done:\n",
    "        state = env.reset()\n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        \n",
    "    if len(replay_buffer) > batch_size:\n",
    "        loss = compute_td_loss(batch_size)\n",
    "        losses.append(loss.item()) #.data[0])\n",
    "        \n",
    "    if frame_idx % 200 == 0:\n",
    "        plot(frame_idx, all_rewards, losses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p><hr></p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Atari Environment</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('common')\n",
    "from common.wrappers import make_atari, wrap_deepmind, wrap_pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "env_id = \"PongNoFrameskip-v4\"\n",
    "env    = make_atari(env_id)\n",
    "env    = wrap_deepmind(env)\n",
    "env    = wrap_pytorch(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CnnDQN(nn.Module):\n",
    "    def __init__(self, input_shape, num_actions):\n",
    "        super(CnnDQN, self).__init__()\n",
    "        \n",
    "        self.input_shape = input_shape\n",
    "        self.num_actions = num_actions\n",
    "        \n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(input_shape[0], 32, kernel_size=8, stride=4),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 64, kernel_size=4, stride=2),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(64, 64, kernel_size=3, stride=1),\n",
    "            nn.ReLU()\n",
    "        )\n",
    "        \n",
    "        self.fc = nn.Sequential(\n",
    "            nn.Linear(self.feature_size(), 512),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(512, self.num_actions)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "    \n",
    "    def feature_size(self):\n",
    "        return self.features(autograd.Variable(torch.zeros(1, *self.input_shape))).view(1, -1).size(1)\n",
    "    \n",
    "    def act(self, state, epsilon):\n",
    "        if random.random() > epsilon:\n",
    "            \n",
    "            with torch.no_grad():\n",
    "                state   = Variable(torch.FloatTensor(np.float32(state)).unsqueeze(0))\n",
    "            q_value = self.forward(state)\n",
    "            action  = q_value.max(1)[1].data[0]\n",
    "        else:\n",
    "            action = random.randrange(env.action_space.n)\n",
    "        return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 84, 84) 6\n"
     ]
    }
   ],
   "source": [
    "print(env.observation_space.shape, env.action_space.n)\n",
    "\n",
    "model = CnnDQN(env.observation_space.shape, env.action_space.n)\n",
    "\n",
    "if USE_CUDA:\n",
    "    model = model.cuda()\n",
    "    \n",
    "optimizer = optim.Adam(model.parameters(), lr=0.00001)\n",
    "\n",
    "replay_initial = 10000\n",
    "replay_buffer = ReplayBuffer(100000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon_start = 1.0\n",
    "epsilon_final = 0.01\n",
    "epsilon_decay = 30000\n",
    "\n",
    "epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f6ef7d8f5f8>]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGKdJREFUeJzt3XtwVOd5x/Hvsyut7kIIiSCQMMYlsbHrq8bGTS9uk9bYTWCmk7R4kibNzdO07i2ZduxJ6zbuP7lNL25IY0+apM3Ece20kxCXDNOJ6TRNYwe5Ng5gSATGRoCNAIEAAbo9/WOP8LLs5UisdHTO/j4zGp199909z9ERP1695+w55u6IiEhypaIuQEREZpeCXkQk4RT0IiIJp6AXEUk4Bb2ISMIp6EVEEk5BLyKScAp6EZGEU9CLiCRcTVQr7ujo8BUrVkS1ehGRWHruueeOunvndF4TWdCvWLGCvr6+qFYvIhJLZvbKdF+jqRsRkYRT0IuIJJyCXkQk4RT0IiIJp6AXEUm4skFvZl82syNmtqPI82ZmD5tZv5m9aGY3V75MERGZqTAj+q8Ca0s8fxewKvi6F/jHyy9LREQqpWzQu/t/A8dLdFkP/ItnPQO0mVlXpQrMt23/cT67ZTcTk7oFoohIGJWYo18GHMh5PBC0XcLM7jWzPjPrGxwcnNHKXnj1BBu37mVkdHxGrxcRqTaVCHor0FZwuO3uj7p7r7v3dnZO6xO8FzTVZT/Me+b8xIxeLyJSbSoR9ANAT87jbuBQBd63oKa6NACnz2tELyISRiWCfhPwvuDsmzXASXc/XIH3Laj5woheQS8iEkbZi5qZ2TeAO4AOMxsA/hKoBXD3LwKbgbuBfmAE+MBsFQs5UzeaoxcRCaVs0Lv7PWWed+D3K1ZRGU0ZzdGLiExH7D4ZOzVHr6kbEZFwYhf0U3P0OhgrIhJO7IK+SQdjRUSmJXZB31CbxgzOjGqOXkQkjNgFfSplNNamNaIXEQkpdkEP2ekbBb2ISDixDPrmuhodjBURCSmWQa8RvYhIeDEN+rQOxoqIhBTPoM9oRC8iElY8g15TNyIiocU26E/rWjciIqHEMuib69K6w5SISEixDPqmuhpGRieY1H1jRUTKimfQZ3RNehGRsOIZ9LpvrIhIaDENet03VkQkrFgG/dQ16XVAVkSkvFgGfWNGNx8REQkrlkHfrDl6EZHQYhn0um+siEh4sQx63TdWRCS8WAZ9kw7GioiEFsugn7pv7OlzCnoRkXJiGfSplNGcqeGUpm5ERMqKZdADtNTXcEojehGRsmIc9LWcOjcWdRkiIvNejINeI3oRkTBiHfTDGtGLiJQV46Cv1YheRCSEGAe9pm5ERMIIFfRmttbM9phZv5ndX+D55Wa21cyeN7MXzezuypd6samDse66y5SISCllg97M0sBG4C5gNXCPma3O6/bnwBPufhOwAfhCpQvN11Jfw9iEc358crZXJSISa2FG9LcC/e6+z91HgceB9Xl9HGgNlhcAhypXYmGt9dnLIOiArIhIaWGCfhlwIOfxQNCW66+A95rZALAZ+INCb2Rm95pZn5n1DQ4OzqDcN7Q21AJonl5EpIwwQW8F2vInxu8Bvuru3cDdwNfM7JL3dvdH3b3X3Xs7OzunX22OlmBEr6AXESktTNAPAD05j7u5dGrmQ8ATAO7+Q6Ae6KhEgcW01E+N6DV1IyJSSpig3wasMrMrzSxD9mDrprw+rwJvAzCza8gG/eXNzZShEb2ISDhlg97dx4H7gC3AS2TPrtlpZg+Z2bqg28eBj5jZduAbwO/4LJ/3qBG9iEg4NWE6uftmsgdZc9sezFneBby1sqWVphG9iEg4sf1kbHOmBjMYVtCLiJQU26C/cPMRTd2IiJQU26AHXe9GRCSMmAe9bj4iIlJOzINeI3oRkXIU9CIiCRfzoNfUjYhIOTEPeo3oRUTKiXnQ1zKsm4+IiJQU66Bva6xlbMI5OzYRdSkiIvNWvIM+uCb9iRHN04uIFBProF8QBP3Jswp6EZFi4h30jRrRi4iUE++gvzCiH424EhGR+SvWQd/WmAE0dSMiUkq8g14HY0VEyop10Ddm0tSkTCN6EZESYh30ZkZbYy0nFPQiIkXFOughe0D2pKZuRESKSkbQa0QvIlJU7IO+rTHDCZ1eKSJSVPyDvqFWZ92IiJQQ+6Bv1dSNiEhJsQ/6tsZaTp0bZ3xiMupSRETmpdgH/dRlEIZ1AxIRkYJiH/RtjbqCpYhIKfEP+obs9W5OjOjMGxGRQmIf9K1T17vRiF5EpKDYB/3U1M2wgl5EpKD4B30woh86o6kbEZFCQgW9ma01sz1m1m9m9xfp85tmtsvMdprZY5Uts7ips26G9KEpEZGCasp1MLM0sBH4VWAA2GZmm9x9V06fVcADwFvdfcjMFs9Wwflq0inaGms5rhG9iEhBYUb0twL97r7P3UeBx4H1eX0+Amx09yEAdz9S2TJLa2/KKOhFRIoIE/TLgAM5jweCtlxvBt5sZj8ws2fMbG2lCgyjvVFBLyJSTNmpG8AKtHmB91kF3AF0A983s+vc/cRFb2R2L3AvwPLly6ddbDHtTRleOTZSsfcTEUmSMCP6AaAn53E3cKhAn2+7+5i7vwzsIRv8F3H3R9291917Ozs7Z1rzJRY1ZzimEb2ISEFhgn4bsMrMrjSzDLAB2JTX51vALwOYWQfZqZx9lSy0lPamDEMjo7jn/6EhIiJlg97dx4H7gC3AS8AT7r7TzB4ys3VBty3AMTPbBWwF/tTdj81W0fkWNmaYmHSGz+rCZiIi+cLM0ePum4HNeW0P5iw78LHga84tas5e7+bYmfMsCD4pKyIiWbH/ZCxAe1MdAEO6sJmIyCWSEfSNwYj+tIJeRCRfMoI+mLrRufQiIpdKRtAHI/rjmroREblEIoK+IZOmoTbNcU3diIhcIhFBD7rejYhIMYkJ+kXNGU3diIgUkJigX6gLm4mIFJSYoF/UlNHplSIiBSQm6Dta6jh6+ryudyMikicxQb+4pY7z45OcOq/r3YiI5EpM0He2ZC+DMHjqfMSViIjML8kJ+uZs0B8ZVtCLiORKTtBPjehPK+hFRHIlJugXt9QDmroREcmXmKBvbaghk05x5NS5qEsREZlXEhP0ZkZnS51G9CIieRIT9JA9l15BLyJysUQF/WIFvYjIJRIV9Jq6ERG5VLKCvrmO4yOjjE1MRl2KiMi8kaigX9xah7vuHSsikitRQT/16VhN34iIvCFZQX/h07E6l15EZEqign5xa/bTsa/rejciIhckK+hb6kgZHD6pEb2IyJREBX1tOkVnSx2HT5yNuhQRkXkjUUEP0LWgQSN6EZEcCQz6eg6f1IheRGRKAoM+O6LXvWNFRLISGPT1jIxOMHxO944VEYGQQW9ma81sj5n1m9n9Jfq9y8zczHorV+L0dLVlT7HU9I2ISFbZoDezNLARuAtYDdxjZqsL9GsB/hB4ttJFTkfXgqmg1wFZEREIN6K/Feh3933uPgo8Dqwv0O+vgc8AkSZs14IGAA6fUNCLiEC4oF8GHMh5PBC0XWBmNwE97v5UBWubkakPTb2mqRsRESBc0FuBtguntJhZCvhb4ONl38jsXjPrM7O+wcHB8FVOQ006xeKWeg5p6kZEBAgX9ANAT87jbuBQzuMW4Drgv8xsP7AG2FTogKy7P+ruve7e29nZOfOqy+hqq+c1Bb2ICBAu6LcBq8zsSjPLABuATVNPuvtJd+9w9xXuvgJ4Bljn7n2zUnEISxc0cEiXQRARAUIEvbuPA/cBW4CXgCfcfaeZPWRm62a7wJnobm9gYOgsk5P60JSISE2YTu6+Gdic1/Zgkb53XH5Zl6dnYSOjE5O8furchbNwRESqVeI+GQuwvL0RgAPHNX0jIpLIoO8Jgv7V4yMRVyIiEr1EBv2ytgbMFPQiIpDQoM/UpOhqrWdAQS8iksygh+z0jUb0IiIJD/oDQwp6EZHEBv3y9kZeHz7PubGJqEsREYlUYoO+pz17/vzAkE6xFJHqltigX37hFMszEVciIhKtxAb9ikVNAOwbVNCLSHVLbNC3N2Voa6xlr4JeRKpcYoPezFjZ0cS+wdNRlyIiEqnEBj3AVZ3N7DuqEb2IVLdEB/3KzmYGT51n+NxY1KWIiEQm4UGvA7IiIokO+qs6mwE0Ty8iVS3RQb+8vZF0yjSiF5Gqluigz9SkWN7eyF6N6EWkiiU66CE7ffOT109FXYaISGQSH/TXdLXw8tEzuriZiFStKgj6ViYd+o9o+kZEqlPig/7qJS0A7Do8HHElIiLRSHzQX7GoifraFLsPa55eRKpT4oM+nTLesqSV3a9pRC8i1SnxQQ9wzZIWXjo8jLtHXYqIyJyriqC/ekkLQyNjHDl1PupSRETmXFUE/TVdrQDsPHQy4kpEROZeVQT9dcsWkDJ44YCCXkSqT1UEfVNdDW9+UwvbD5yIuhQRkTlXFUEPcGNPG9sHTuiArIhUnaoJ+ht62jgxMsYrx0aiLkVEZE6FCnozW2tme8ys38zuL/D8x8xsl5m9aGbfM7MrKl/q5bmxpw2A7QOavhGR6lI26M0sDWwE7gJWA/eY2eq8bs8Dve5+PfBN4DOVLvRyrVrcTENtmudfVdCLSHUJM6K/Feh3933uPgo8DqzP7eDuW919ak7kGaC7smVevpp0iuu7F/DcK0NRlyIiMqfCBP0y4EDO44GgrZgPAd+9nKJmy20rF7Hz0EndLFxEqkqYoLcCbQVPXTGz9wK9wGeLPH+vmfWZWd/g4GD4Kitkzcp2Jh369h+f83WLiEQlTNAPAD05j7uBQ/mdzOztwCeAde5e8FoD7v6ou/e6e29nZ+dM6r0sNy9fSCad4pl9CnoRqR5hgn4bsMrMrjSzDLAB2JTbwcxuAh4hG/JHKl9mZdTXprmxp41n9h2LuhQRkTlTNujdfRy4D9gCvAQ84e47zewhM1sXdPss0Aw8aWYvmNmmIm8XuTUr29lxUPP0IlI9asJ0cvfNwOa8tgdzlt9e4bpmze1XdfDw0/38cO8x7rx2SdTliIjMuqr5ZOyU3hULaamrYevueTvDJCJSUVUX9LXpFL/w5g627jmi696ISFWouqAH+JWr38Trw+fZeUi3FxSR5KvKoL/jLZ2YoekbEakKVRn0Hc113NjTxpZdr0VdiojIrKvKoAf49Z/tYsfBYfYNno66FBGRWVW1Qf+O65diBt/ZfjjqUkREZlXVBv2SBfXcuqKdTdsP6uwbEUm0qg16gHU3LmXv4Bl2HNTZNyKSXFUd9O+4fin1tSke+9ErUZciIjJrqjroFzTUsu6GpXz7hUO69o2IJFZVBz3Ae9dcwcjoBN96/mDUpYiIzIqqD/rru9u4vnsBX/nBfiYmdVBWRJKn6oMe4KO/dBUvHz3Df/xYp1qKSPIo6IE7r13CzyxuZuPT/UxqVC8iCaOgB1Ip4/d/+Sr2vH5Ko3oRSRwFfWDdDcu4pquVT313N+fGJqIuR0SkYhT0gXTK+It3XMPBE2f50vf3RV2OiEjFKOhz/NxVHdx13RL+4el++o/oYmcikgwK+jyfXH8tDZk0H39yO+MTk1GXIyJy2RT0eRa31PPX669j+4ET/M1//iTqckRELpuCvoB33rCUe27t4Qv/tZf/eFFn4YhIvCnoi/jkuuu45YqFfPzJF3hm37GoyxERmTEFfRGZmhSP/PYtdC9s5INf3Ubf/uNRlyQiMiMK+hI6mut47MO38abWet7zpWfZrA9TiUgMKejLWNxaz5O/ezvXLm3l977+f3xuyx7GdDaOiMSIgj6EjuY6HvvIGt59Szef39rPb3zhf9lx8GTUZYmIhKKgD6m+Ns1n330DX3zvzRw8cZZ3fv5/+NMnt3Pg+EjUpYmIlFQTdQFxs/a6Lm6/qoMvbO3nKz/Yz7/93wB3XruE99x2BbdftYh0yqIuUUTkIuYezWV5e3t7va+vL5J1V8prJ8/xzz/cz9efeYXhc+N0NNdx13VL+IVVHdy2chELGmqjLlFEEsbMnnP33mm9RkF/+c6NTfD07iN8Z/shtu45wrmxSVIGVy9p5dqlraxe2srVS1pZvqiRJa31GvWLyIzNWtCb2Vrg74E08CV3/1Te83XAvwC3AMeA33L3/aXeM0lBn+v8+AQvvHqCH+w9xvOvDvHS4WGOnh698HxNylja1kDXgnoWNWdY2JhhUVOGhU0ZFjTU0phJ05CpoaE2HSxnv9emU9SkjJqp7ykjnTLM9J+GSDWZSdCXnaM3szSwEfhVYADYZmab3H1XTrcPAUPu/jNmtgH4NPBb0ykkKepq0ty2chG3rVx0oe3IqXP85LXTHBga4cDxEQaGznL45Fn2vHaKoZExhkZGmekfVlOBX5tOBcEPBpjZG98vtIFxcR+m2g1Sua+BbKd5YJ6UMS/+U42+AqmEP3zbKt55w9I5W1+Yg7G3Av3uvg/AzB4H1gO5Qb8e+Ktg+ZvA583MPKp5oXlmcUs9i1vqiz4/MekMnx3j5Nkxzo5NMDI6wdnRCUZGxzk7ll0em5hkfNIZn/Dg+yRjk87E5ORFbQ64g+PBd3APli9pzz4mp99kzvJ8MD+qYF4U4vOhCKmIuT5+FybolwEHch4PALcV6+Pu42Z2ElgEHK1EkUmXThkLg+kbEZFKC3MefaG/FvOHFmH6YGb3mlmfmfUNDg6GqU9ERC5TmKAfAHpyHncDh4r1MbMaYAFwyVXA3P1Rd+91997Ozs6ZVSwiItMSJui3AavM7EozywAbgE15fTYB7w+W3wU8rfl5EZH5oewcfTDnfh+whezplV92951m9hDQ5+6bgH8CvmZm/WRH8htms2gREQkv1CUQ3H0zsDmv7cGc5XPAuytbmoiIVIIuaiYiknAKehGRhFPQi4gkXGQXNTOzQeCVGb68g+r7MJa2uTpom6vD5WzzFe4+rfPTIwv6y2FmfdO9qE/caZurg7a5Osz1NmvqRkQk4RT0IiIJF9egfzTqAiKgba4O2ubqMKfbHMs5ehERCS+uI3oREQkpdkFvZmvNbI+Z9ZvZ/VHXU46Z9ZjZVjN7ycx2mtkfBe3tZvafZvbT4PvCoN3M7OFg+140s5tz3uv9Qf+fmtn7c9pvMbMfB6952IJbIRVbxxxue9rMnjezp4LHV5rZs0E9/xpcJA8zqwse9wfPr8h5jweC9j1mdmdOe8Hfg2LrmKPtbTOzb5rZ7mB/3570/WxmfxL8Xu8ws2+YWX3S9rOZfdnMjpjZjpy2yPZrqXUUlb37UDy+yF5UbS+wEsgA24HVUddVpuYu4OZguQX4CbAa+Axwf9B+P/DpYPlu4Ltkr/G/Bng2aG8H9gXfFwbLC4PnfgTcHrzmu8BdQXvBdczhtn8MeAx4Knj8BLAhWP4i8NFg+feALwbLG4B/DZZXB/u4Drgy2PfpUr8HxdYxR9v7z8CHg+UM0Jbk/Uz2hkMvAw05P/vfSdp+Bn4RuBnYkdMW2X4tto6S2zBX/wgq9AO/HdiS8/gB4IGo65rmNnyb7P139wBdQVsXsCdYfgS4J6f/nuD5e4BHctofCdq6gN057Rf6FVvHHG1nN/A94FeAp4JfyqNATf6+JHtl1NuD5Zqgn+Xv36l+xX4PSq1jDra3lWzoWV57Yvczb9xZrj3Yb08BdyZxPwMruDjoI9uvxdZRqv64Td0Uuq3hsohqmbbgT9WbgGeBN7n7YYDg++KgW7FtLNU+UKCdEuuYC38H/BkwGTxeBJxw9/ECdV50K0pg6laU0/1ZlFrHbFsJDAJfsex01ZfMrIkE72d3Pwh8DngVOEx2vz1HsvfzlCj367RzMG5BH+qWhfORmTUD/wb8sbsPl+paoM1n0B4ZM3sHcMTdn8ttLtDVyzwXp59FDdk/7//R3W8CzpD9c7uYOG1bQcGc8Xqy0y1LgSbgrgJdk7Sfy5mLbZn2a+IW9GFuazjvmFkt2ZD/urv/e9D8upl1Bc93AUeC9mLbWKq9u0B7qXXMtrcC68xsP/A42embvwPaLHuryfw6i92Kcro/i6Ml1jHbBoABd382ePxNssGf5P38duBldx909zHg34GfI9n7eUqU+3XaORi3oA9zW8N5JTiC/k/AS+7+NzlP5d5+8f1k5+6n2t8XHFlfA5wM/mzbAvyamS0MRlK/RnZe8jBwyszWBOt6X957FVrHrHL3B9y9291XkN1HT7v7e4CtZG81mV9PsVtRbgI2BGdrXAmsInvgquDvQfCaYuuYVe7+GnDAzN4SNL0N2EWC9zPZKZs1ZtYY1DS1zYndzzmi3K/F1lHcXBy0qfBBkbvJnrmyF/hE1PWEqPfnyf5Z9SLwQvB1N9l5xu8BPw2+twf9DdgYbN+Pgd6c9/og0B98fSCnvRfYEbzm87zxQbiC65jj7b+DN866WUn2H3A/8CRQF7TXB4/7g+dX5rz+E8F27SE4G6HU70GxdczRtt4I9AX7+ltkz65I9H4GPgnsDur6GtkzZxK1n4FvkD0GMUZ2NP2hKPdrqXUU+9InY0VEEi5uUzciIjJNCnoRkYRT0IuIJJyCXkQk4RT0IiIJp6AXEUk4Bb2ISMIp6EVEEu7/AaGX6v0bqYBpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([epsilon_by_frame(i) for i in range(1000000)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwAAAAE/CAYAAADxMqTfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xl8lOW5//HPlWWSDEsSIOyJiKAoyJrg0lpbtS61ihsCLu05p7+617baU7XtqbZHW23t0VrrwmnPObYq4Fp3Uan7BoQdAUFAEtYA2SCELHP//pgnGiD7zGSSeb7v12tezLPfSUjmuZ77uq/bnHOIiIiIiIg/JMW7ASIiIiIi0nkUAIiIiIiI+IgCABERERERH1EAICIiIiLiIwoARERERER8RAGAiIiIiIiPKABIQGZ2lJktNrNKM7s+3u2R2DKzjWZ2WrzbISKSKPR3VRKdAoDE9FPgLedcL+fcffFuTGNmdqSZPWdmJWa228zmmtlRB+3zYzPbZmblZvY/ZpbWaNswM3vTzKrMbPXBf6AjOdaPzOx4M3vd+1mUmNmTZjao0XYzs7vMbJf3+p2ZWTPnOtvM3jOzMu9n8N9m1qvzvhoRERFpCwUAiekwYGVzG80suRPbcrAs4HngKGAAMB94rmGjmZ0B3AycCgwDhgO/anT8LGAx0Bf4OfCUmeVEemx7mFlKe4+JhhhdNxuYSfj7dRhQCfxvo+1XAOcB44CxwLeBK5s5VyZwOzAYOBoYCvw+Bm0WERGRSDjn9EqgF/BPoB6oBvYARwL/BzwIvAzsBU4DziZ8M1wBFAG3NTrHMMAB/+ptKwWuAgqAZUAZcP9B1/03YJW371zgsDa2t493rb7e8uPAbxptPxXY5r0/EtgP9Gq0/V3gqkiPbUM7NwI3eV//fiCF8I3u00AJsAG43ts3HdgH9POWfwHUAb295duBe733bfk5fA/YBLzjrb8c+BzYRTiQ2QicFqX/PxOBykbLHwBXNFr+HvBRG891AbA83r8Teumll17tfTX8XQXSgHuBLd7rXiDN26cf8KL3mbjb+0xJ8rbdBGwm/FBlDXBqvL8mvfRq/FIPQIJxzp1C+I/Qdc65ns65T71NlwB3AL2A9wgHAt8h/ET+bOBqMzvvoNMdB4wEphH+o/dzwn8QRwMXm9nJAN5xPyN8w5fjXX9WG5v8NcI36bu85dHA0kbblwIDzKyvt229c67yoO2jo3BsW8wg/L3KAkLAC945hhAONn5kZmc456qBBcDJjb7Gz4GvNFp+23vflp/DyYSfqJ9hZscQDuYuJxyA9CX8pB0AM/uqmZW142s62Nc4sPeoqe9pW79nB59LRKS7+TlwPDCecE/oZMIPdQBuBIoJf+4NIPw56Ly01uuAAudcL+AMwgGFSJehAMA/nnPOve+cCznnqp1zbznnlnvLywjfsJ980DH/6e37GuEb1VnOuR3Ouc2Eb/InePtdCfzWObfKOVcH/AYYb2aHtdQgMxsK/Bm4odHqnkB5o+WG972a2NawvSHPPJJj2+I+51yRc24f4d6QHOfcr51zNc659cB/A9O9fd8GTvbSdsYC93nL6d6x7wK08edwm3Nur3fdi4AXnXPvOOf2A/9BOBjBO997zrmsdnxNXzCzscAvgX9vtLqp72nP5sYBNDrXN4HveucTEemuLgV+7X32lRBOK73c21YLDCLc413rnHvXOecI98KnAceYWapzbqNz7rO4tF6kGQoA/KOo8YKZHecNiC0xs3LCKT79Djpme6P3+5pY7um9Pwz4ozf4s6Er1Ag/GW+Sl3v/GvCAc65xb8EeoHej5Yb3lU1sa9je8FQ/kmPbovH38DBgcMPX7H3dPyP8FAjCAcDXCafULAdeJ3xjfzywzjm3E9r8c2h83cGNl51zewmnArXKzPLMbE/D66BtI4BXgB86595ttKmp7+ke70OuuescTzgd66JGPVAiIt3RYMI9uA0+99ZBeIzTOuA1M1tvZjcDOOfWAT8CbgN2mNlsMxuMSBeiAMA/Dr5he5zwYNxc51wm8BDhm/aOKAKudM5lNXplOOc+aGpnM8smfPP/vHPujoM2ryTczdpgHLDdSxFaCQw/qLLMOL5MM4nk2LZo/D0sAjYc9DX3cs59y9v+AeGBzucDbzvnPgHyCKf5vN3oPG35OTS+7lYgt2HBzIKE04Bab7xzm7y0sJ7OuYbgDa+n5g3CPT5/P+iwpr6nLQ0wn+B9Pf/mnJvXlnaJiHRhWwg/8GmQ563DOVfpnLvROTccOAe4wcxO9bY97pz7qnesA+7q3GaLtEwBgH/1AnY756rNbDLhMQId9RBwi5mNBjCzTDOb2tSOZtab8CDh951zNzexy9+A75nZMV6g8AvCg5jxniYvAW41s3QzO59wes3TUTi2veYDFWZ2k5llmFmymY0xswLvelVAIXAtX97wf0A4XapxANDen8NTwLe9XP8A8Gsi+D02syGEB47/2Tn3UBO7/I3wh9oQ7wnWjXjf0ybONQZ4FfiBc+6FjrZJRKQLmQX8wsxyzKwf4bTGRwHM7NtmNsJLiawgnPpTb+G5eE7xylBXE+4xr49T+0WapADAv64Bfm1mlYT/oD3R0RM5554l/HRjtplVACuAs5rZ/XzCOfD/2jgdxczyvHO9CvwOeJNwV+vnwK2Njp8O5BOuNnQn4TSTkkiPNbNLzazNvQHOuXrCT3zGE64AtBP4C+FSmA3eBlIJBwsNy72Adxrt066fg3NuJeGg4nHCvQGlhAeh4X0dJx2c3tOK/0e4XOqtzaQHPUx4sPNywj/Xl7x1DdfbY2YneYs3Eh4M99dG59IgYBHpzm4HFhKuALccWOStg3CRjDcIp0p+SDil9S3C+f93Ev5c2Ab0J5wiKtJlWAupvCIiIiIikmDUAyAiIiIi4iMKAEREREREfEQBgIiIiIiIjygAEBERERHxEQUAIiIiIiI+khLvBjTWr18/N2zYsHg3Q0SkSyosLNzpnMuJdzviSZ8TIiJNa89nRJcKAIYNG8bChQvj3QwRkS7JzD6PdxviTZ8TIiJNa89nhFKARERERER8RAGAiIiIiIiPKAAQEREREfERBQAiIiIiIj6iAEBERERExEcUAIiIiIiI+IgCABERERERH4lZAGBm483sIzNbYmYLzWxyrK4lIiIiIiJtE8segN8Bv3LOjQd+6S2LiIiIiEgcxTIAcEBv730msCVWF1peXM7jH2+K1elFRESataVsH2u3V8a7GSIibZYSw3P/CJhrZncTDjRObGonM7sCuAIgLy+vQxd6Y9V2/jhvLRfnDyUlWcMaRESk85x45z8B2Hjn2XFuiYhI20R0t2xmb5jZiiZeU4CrgR8753KBHwN/beoczrmZzrl851x+Tk5Oh9qRFUwFoKK6rmNfiIiIiIiIT0TUA+CcO625bWb2N+CH3uKTwF8iuVZLGgKAsqoa+vQIxOoyIiIiIiLdXizzZbYAJ3vvTwHWxupCWcHwTX9pVW2sLiEiIiIikhBiOQbg+8AfzSwFqMbL84+FrIxwD0D5vppYXUJEREREJCHELABwzr0HTIrV+Rtr6AEoUw+AiIiIiEiLEqJkTkMPgAIAEREREZGWJUQA0DsjFbPwIGAREREREWleQgQAyUlG7/RUyvapB0BEREREpCUJEQBAuBSoUoBERERERFqWOAFARiqlSgESEREREWlR4gQAwQDlSgESEREREWlRAgUASgESEekMZvY/ZrbDzFY0WtfHzF43s7Xev9neejOz+8xsnZktM7OJjY75rrf/WjP7bjy+FhERP0qcACAjVVWAREQ6x/8BZx607mZgnnNuJDDPWwY4Cxjpva4AHoRwwADcChwHTAZubQgaREQkthInAAgGqKiuo64+FO+miIgkNOfcO8Dug1ZPAR7x3j8CnNdo/d9c2EdAlpkNAs4AXnfO7XbOlQKvc2hQISIiMZBAAUB4MrCK6ro4t0RExJcGOOe2Anj/9vfWDwGKGu1X7K1rbv0hzOwKM1toZgtLSkqi3nAREb9JuABAaUAiIl2KNbHOtbD+0JXOzXTO5Tvn8nNycqLaOBERP0qcACAjAKDJwERE4mO7l9qD9+8Ob30xkNtov6HAlhbWi4hIjCVOAKAeABGReHoeaKjk813guUbrv+NVAzoeKPdShOYCp5tZtjf493RvnYiIxFhKvBsQLVlBrwdApUBFRGLKzGYBXwf6mVkx4Wo+dwJPmNn3gE3AVG/3l4FvAeuAKuBfAZxzu83sP4EF3n6/ds4dPLBYRERiIHECgIyGHgAFACIiseScm9HMplOb2NcB1zZznv8B/ieKTRMRkTZImBSg3hmpmGkMgIiIiIhISxImAEhOMnqnazIwEREREZGWJEwAAOGBwEoBEhERERFpXmIFABmpSgESEREREWlBYgUAwQDlSgESEREREWlWggUAqZQqBUhEREREpFmJFQBkaBCwiIiIiEhLEioAyAwGqKiuoz7k4t0UEREREZEuKaECgOxgeDKwcg0EFhERERFpUkIFAFnBhtmAlQYkIiIiItKUxAoAMgKAZgMWEREREWlOQgUAmQ0pQKoEJCIiIiLSpIQKALKD4R6AUqUAiYiIiIg0KaECgKyMhjEA6gEQEREREWlKRAGAmU01s5VmFjKz/IO23WJm68xsjZmdEVkz26Z3RipmGgMgIiIiItKclAiPXwFcADzceKWZHQNMB0YDg4E3zOxI51x9hNdrUXKS0Ts9lXKlAImIiIiINCmiHgDn3Crn3JomNk0BZjvn9jvnNgDrgMmRXKutsoKplCoFSERERESkSbEaAzAEKGq0XOytO4SZXWFmC81sYUlJScQXzspIVQqQiIiIiEgzWk0BMrM3gIFNbPq5c+655g5rYp1rakfn3ExgJkB+fn6T+7RHZjCgFCARERERkWa0GgA4507rwHmLgdxGy0OBLR04T7tlB1P5fNfezriUiIiIiEi3E6sUoOeB6WaWZmaHAyOB+TG61gGyMlIp3aseABERERGRpkRaBvR8MysGTgBeMrO5AM65lcATwCfAq8C1sa4A1CAzGKCiuo76UMTZRCIiIiIiCSeiMqDOuWeBZ5vZdgdwRyTn74iGycAq9tWS3SPQ2ZcXEREREenSEmomYIDsHuEAoFQDgUVEREREDpFwAUBWRvipv0qBioiIiIgcKuECgMxguAegXJOBiYiIiIgcIuECgOxgQw+AUoBERERERA6WcAFAwyDg0r3qARAREREROVjCBQC9vQBAYwBERERERA6VcAFAcpLROz2FclUBEhERERE5RMIFAADZPQLqARARERERaUJCBgBZGamUqgqQiIiIiMghEjIAyAwGlAIkIiIiItKEhAwAsjJSlQIkIiIiItKEhAwAsoOplCkFSERERETkEAkZAGQGA1RU11IfcvFuioiIiIhIl5KQAUBWRirOQYXSgEREREREDpCQAUB2D00GJiISD2b2YzNbaWYrzGyWmaWb2eFm9rGZrTWzOWYW8PZN85bXeduHxbf1IiL+kJABQFZGAIAyVQISEek0ZjYEuB7Id86NAZKB6cBdwD3OuZFAKfA975DvAaXOuRHAPd5+IiISYwkZAGQGvR4ADQQWEelsKUCGmaUAQWArcArwlLf9EeA87/0Ubxlv+6lmZp3YVhERX0rIACAroyEFSD0AIiKdxTm3Gbgb2ET4xr8cKATKnHN13m7FwBDv/RCgyDu2ztu/b2e2WUTEjxIyAMgONqQAqQdARKSzmFk24af6hwODgR7AWU3s2lCiramn/YeUbzOzK8xsoZktLCkpiVZzRUR8KyEDgN5eD0CpAgARkc50GrDBOVfinKsFngFOBLK8lCCAocAW730xkAvgbc8Edh98UufcTOdcvnMuPycnJ9ZfQ4eV7lWvs4h0DwkZACQnGb3TUyjXIGARkc60CTjezIJeLv+pwCfAm8BF3j7fBZ7z3j/vLeNt/6dzrttO4HLRQx/EuwkiIm2SkAEAQFYwoDKgIiKdyDn3MeHBvIuA5YQ/Y2YCNwE3mNk6wjn+f/UO+SvQ11t/A3Bzpzc6ij4r2RvvJoiItElK67t0T9nBVI0BEBHpZM65W4FbD1q9HpjcxL7VwNTOaJeIiHwpYXsAMoMBzQMgIiIiInKQhA0AsjJSlQIkIiIiInKQhA0AlAIkIiIiInKohA0AMoMBKqprqQ9124ISIiIiIiJRl7ABQFZGKs5BhdKARERERES+kLgBQDA8GZjGAYiIiIiIfCmiAMDMpprZSjMLmVl+o/XfNLNCM1vu/XtK5E1tn+xgAECVgEREREREGol0HoAVwAXAwwet3wmc45zbYmZjgLnAkAiv1S6Z6gEQERERETlERAGAc24VQHjG9wPWL260uBJIN7M059z+SK7XHlkZXgCgHgARERERkS90xhiAC4HFzd38m9kVZrbQzBaWlJRE7aJZX6QAqQdARERERKRBqz0AZvYGMLCJTT93zj3XyrGjgbuA05vbxzk3E5gJkJ+fH7WanZlf9AAoABARERERadBqAOCcO60jJzazocCzwHecc5915ByRSE4yeqenUK4xACIiIiIiX4hJCpCZZQEvAbc4596PxTXaIisYoFRjAEREREREvhBpGdDzzawYOAF4yczmepuuA0YA/2FmS7xX/wjb2m7ZwVSlAImIiIiINBJpFaBnCaf5HLz+duD2SM4dDZnBgMqAioiIiIg0krAzAUO4FKjKgIqIiIiIfCmxAwClAImIiIiIHCDBA4AAFdW11IeiVl1URERERKRbS+wAICMV56CyWr0AIiIiIiKQ6AFAMDwZWKnSgEREREREAJ8EABoILCIiIiISluABQABApUBFRERERDyJHQBkhHsAypUCJCIiIiICJHoA4PUAlCoFSEREREQESPAAIDOjYQyAegBERERERCDBA4DkJKN3egrlGgMgIiIiIgIkeAAA4TQgVQESEREREQnzQQCQqnkAREREREQ8PggAAioDKiIiIiLiSfwAICOVcqUAiYiIiIgAfggAgqnqARARERER8SR+AJCRSvm+WupDLt5NERERERGJu8QPAIIBnIPKavUCiIiIiIj4IADQZGAiIiIiIg18EwCUaiCwiIiIiEjiBwCZGQEADQQWEREREcEHAUC21wNQrhQgEREREZHEDwCygl4PgFKARERizsyyzOwpM1ttZqvM7AQz62Nmr5vZWu/fbG9fM7P7zGydmS0zs4nxbr+IiB8kfADQOz0FgFL1AIiIdIY/Aq8650YB44BVwM3APOfcSGCetwxwFjDSe10BPNj5zRUR8Z+EDwBSkpPonZ5CucYAiIjElJn1Br4G/BXAOVfjnCsDpgCPeLs9ApznvZ8C/M2FfQRkmdmgTm62iIjvJHwAAOE0IKUAiYjE3HCgBPhfM1tsZn8xsx7AAOfcVgDv3/7e/kOAokbHF3vrREQkhnwSAKSqCpCISOylABOBB51zE4C9fJnu0xRrYt0h07ab2RVmttDMFpaUlESnpSIiPuaLACAzI1VjAEREYq8YKHbOfewtP0U4INjekNrj/buj0f65jY4fCmw5+KTOuZnOuXznXH5OTk7MGi8i4he+CACygwHKlQIkIhJTzrltQJGZHeWtOhX4BHge+K637rvAc97754HveNWAjgfKG1KFREQkdiIKAMxsqpmtNLOQmeU3sT3PzPaY2U8iuU6klAIkItJpfgA8ZmbLgPHAb4A7gW+a2Vrgm94ywMvAemAd8N/ANZ3fXBER/0mJ8PgVwAXAw81svwd4JcJrRCwrI5XyfbWEQo6kpKZSTkVEJBqcc0uAQx4IEe4NOHhfB1wb80aJiMgBIgoAnHOrAMwOvak2s/MIP9nZG8k1oiEzGMA5qKiu/WJiMBERERERP4rJGACv7NtNwK/asG/MqztkB1MBKNNAYBERERHxuVYDADN7w8xWNPGa0sJhvwLucc7tae38nVHdIashANA4ABERERHxuVZTgJxzp3XgvMcBF5nZ74AsIGRm1c65+ztwrohlZoTTfkpVCUhEREREfC7SQcBNcs6d1PDezG4D9sTr5h++TAEqVwqQiIiIiPhcpGVAzzezYuAE4CUzmxudZkVXw8DfMvUAiIiIiIjPRVoF6Fng2Vb2uS2Sa0RD7/Twl6kxACIiIiLid76YCTglOYle6SmqAiQiIiIivueLAAAgOxhQCpCIiIiI+J5vAoCsYKpSgERERETE93wTAGRmpCoFSERERER8zzcBQJZSgERERERE/BMAZCsFSERERETEPwFAVkYq5ftqCYVcvJsiIiIiIhI3vgkAMoMBnIPK6rp4N0VEREREJG58EwBkB1MBKNU4ABERERHxMd8EAFleAKBxACIiIiLiZ74JADIzAgCqBCQiIiIivuabAKChB6BcPQAiIiIi4mO+CQCyg+EegNK96gEQEREREf/yTQDQOz0F0BgAERGJnYUbd8e7CSIirfJNAJCSnESv9BTKqhQAiIhIbFz00IfxboKISKt8EwBAeByABgGLiIiIiJ/5KgDIDgaUAiQiIiIivuarACAzI1UpQCIiIiLia74KALKCAZUBFRERERFf81UAkB1MpVRjAERERETEx3wVAGRlpFK+r5ZQyMW7KSIiIiIiceGrACAzGMA5qKyui3dTRERERETiwlcBQFZGKgBl+5QGJCIiIiL+5KsAILtHOAAoVSUgEREREfEpXwUAmRkBAE0GJiIiIiK+5asAICsY7gFQKVARERER8St/BQANYwCUAiQiIiIiPuWrACAzo2EMgFKAREQkcos3lca7CSIi7earACAlOYle6SnqARARkah4adnWeDdBRKTdIgoAzGyqma00s5CZ5R+0bayZfehtX25m6ZE1NTqygqkaAyAiIiIivpUS4fErgAuAhxuvNLMU4FHgcufcUjPrC3SJu+7sYEBVgERERETEtyLqAXDOrXLOrWli0+nAMufcUm+/Xc65+kiuFS2ZGamaB0BEJIbMLNnMFpvZi97y4Wb2sZmtNbM5Zhbw1qd5y+u87cPi2W4REb+I1RiAIwFnZnPNbJGZ/bS5Hc3sCjNbaGYLS0pKYtScL2UFA0oBEhGJrR8Cqxot3wXc45wbCZQC3/PWfw8odc6NAO7x9utWzOLdAhGR9ms1ADCzN8xsRROvKS0clgJ8FbjU+/d8Mzu1qR2dczOdc/nOufycnJwOfRHtkZWRqhQgEZEYMbOhwNnAX7xlA04BnvJ2eQQ4z3s/xVvG236qt7+IiMRQq2MAnHOndeC8xcDbzrmdAGb2MjARmNeBc0VVtjcIOBRyJCXpc0ZEJMruBX4K9PKW+wJlzrk6b7kYGOK9HwIUATjn6sys3Nt/Z+c1V0TEf2KVAjQXGGtmQW9A8MnAJzG6VrtkBgOEHFRW17W+s4iItJmZfRvY4ZwrbLy6iV1dG7Y1Pm+npoqKiCS6SMuAnm9mxcAJwEtmNhfAOVcK/BewAFgCLHLOvRRpY6Phi9mA9ykNSEQkyr4CnGtmG4HZhFN/7gWyvIdBAEOBLd77YiAXvqgelwnsPviknZ0qKiKS6CKtAvSsc26ocy7NOTfAOXdGo22POudGO+fGOOeaHQTc2bKCXgCgSkAiIlHlnLvF+0wYBkwH/umcuxR4E7jI2+27wHPe++e9Zbzt/3TOHdID0JVpyIKIdEe+mgkYwlWAAEo1EFhEpLPcBNxgZusI5/j/1Vv/V6Cvt/4G4OY4tU9ExFcinQis22noAVApUBGR2HHOvQW85b1fD0xuYp9qYGqnNkxERHzYA5ChFCARERER8S/fBQCZCgBERCRKNAJARLoj3wUAKclJ9EpP0RgAEREREfEl3wUAEB4HoDEAIiIiIuJH/gwAMgKUqQdARERioFwppiLSxfkzAAimUqYeABERiVQTgwB+/9rqzm+HiEg7+DQACGgQsIiIxER9KN4tEBFpmT8DgIxUpQCJiIiIiC/5MwDwBgGHQt1qxnkREekG9NkiIl2dTwOAACEHlfvr4t0UERHpxqyJQQAbd+2NQ0tERNrOnwHAF5OBKQ1IRERERPzFnwFAULMBi4iIiIg/+TQACACoFKiIiIiI+I5PAwClAImIiIiIP/kzAMhQCpCIiETOmpgIrEYTAYhIF+fLACBTAYCIiMTI4k1l8W6CiEiLfBkApCQn0Ss9hbJ9SgESEREREX/xZQAA4XEA6gEQEREREb/xbwCQEdAgYBERiUgTQwBERLo8/wYAwVSVARURERER3/FxABCgXClAIiISARfvBoiIdIB/A4CMVEqVAiQiIiIiPuPfACCYSvm+WkIhPb8REZGO0RgAEemOfBwABAg5qNxfF++miIiIiIh0Gv8GAN5kYBoHICIiIiJ+4t8AIBgOADQOQERERET8xPcBgEqBiohIR5kGAYhINxRRAGBmU81spZmFzCy/0fpUM3vEzJab2SozuyXypkZXVjAAoMnARERERMRXIu0BWAFcALxz0PqpQJpz7lhgEnClmQ2L8FpR1TAGoExjAERERETERyIKAJxzq5xza5raBPQwsxQgA6gBKiK5VrRlKgCQFoRCjkWbSuPdDBEREZGoi9UYgKeAvcBWYBNwt3Nud1M7mtkVZrbQzBaWlJTEqDmHSklOoldaCmX7lAIkh3qysIgLHviAdz7tvP+TIiIiIp2h1QDAzN4wsxVNvKa0cNhkoB4YDBwO3Ghmw5va0Tk30zmX75zLz8nJ6dAX0VFZPVJVBlSa9Pj8IgBmL9gU55aISFdmmgpMRLqhlNZ2cM6d1oHzXgK86pyrBXaY2ftAPrC+A+eKmayMgMqAyiFWba1gaVEZA3qn8fon29m5Zz/9eqbFu1kiIiIiURGrFKBNwCkW1gM4Hlgdo2t1WFYwVWVA5RBzFhQRSEniz5dMpLbe8cyi4ng3SURERCRqIi0Der6ZFQMnAC+Z2Vxv05+BnoSrBC0A/tc5tyyilsZAVjCgFCA5QHVtPc8sKuasMQPJH9aH/MOymb2gCOdcvJsmIiIiEhWRVgF61jk31DmX5pwb4Jw7w1u/xzk31Tk32jl3jHPu99FpbnRlZagHQA706optVFTXMa0gF4BpBbmsL9nLgo2qCCQih9JEYCLSHfl2JmDwUoCqagjK9tqiAAAgAElEQVSF9HRXwmbN38RhfYMcf3hfAM4eO4heaSnMnq/BwCIiIpIYfB0AZGakEnJQub8u3k2RLmB9yR4+3rCbaQW5JCWFH+sFAymcO34wLy3fSrl6i0RaZGa5ZvamNwP8SjP7obe+j5m9bmZrvX+zvfVmZveZ2TozW2ZmE+P7FYiI+IOvA4DsYABA4wAEgDkLi0hOMi6aOPSA9TMm57G/LsRzSzbHqWUi3UYdcKNz7mjCxR+uNbNjgJuBec65kcA8bxngLGCk97oCeLDzmyzSvPKqWobd/BKvf7I93k0RiSpfBwBZQW82YE0G5nu19SGeLizm1FH96d87/YBtY4ZkMnpwb2bN12BgkZY457Y65xZ57yuBVcAQYArwiLfbI8B53vspwN9c2EdAlpkN6uRmR0RDABLbmu2VAMx857M4t0QkuhQAAKXqAfC9eau2s3NPDdMn5za5fXpBLqu2VrBic0Unt0ykezKzYcAE4GNggHNuK4SDBKC/t9sQoKjRYcXeOpEuRc9+JNH4OgDIzAinAJVpMjDfm72giIG90zn5yP5Nbj93/BDSU5OYpZmBRVplZj2Bp4EfOedaipqbeoB+yK2WmV1hZgvNbGFJSUm0minSKlV5kkTl6wAg2+sB0OBOf9tcto+3Py3h4vyhJCc1/dc+MyOVbx07iOeXbKGqRoPGRZpjZqmEb/4fc849463e3pDa4/27w1tfDDTudhsKbDn4nM65mc65fOdcfk5OTuwaL9IMdQBIovF1AJCZ4Y0BUAqQrz25MJyBMDW/6fSfBjMm57Fnfx0vLtvaGc0S6XbMzIC/Aqucc//VaNPzwHe9998Fnmu0/jteNaDjgfKGVKFuQ4+IE5p+upKofB0ApCQn0SsthVKlAPlWfcjxxIIivjqiH7l9gi3um39YNkfk9GDOgqIW9xPxsa8AlwOnmNkS7/Ut4E7gm2a2FvimtwzwMrAeWAf8N3BNHNosIuI7KfFuQLxl9UhVGVAfe3dtCVvKq/nFt49pdV8zY3pBHne8vIpPt1dy5IBendBCke7DOfcezT80PbWJ/R1wbUwbJRIFqgAnicbXPQAAWRkByjQGwLdmzy+iT48Apx09oE37XzBxCKnJpl4AEREfUIaXJCoFAMFUpQD5VEnlft5YtZ0LJw4hkNK2X4W+PdM4/ZiBPLOomP119TFuoYiIdAV6/i+JxvcBQGaGUoD86ulFxdSFHNMK8tp13LSCXEqranltZfRnhqyurWfdjj1RP29Xtb2imp179se7GZ3mmUXFKjucYPSAONHpJyyJyfcBwJCsDIpKq1hWXBbvpkgncs4xZ0ERBcOyGdG/Z7uO/eqIfgzJymB2lOcEqA85vv+3hZxx7zsU7a6K6rm7oqLdVZxx7zt875GF8W5Kp1ixuZwbnljKc0sOqXIpIl2chgBIovF9AHDlyUfQv1c6Vz+6iN179WTOLz7esJsNO/cyvZ1P/wGSkoxpBbm8v24Xm3ZF70b9ntc/5d21O6kPuS9Kkyaq6tp6rnq0kLKqWpYWlbFyS3m8mxRzs+ZvIi0lifPGa6LbRLKj0j89WCKSOHwfAPTpEeChyyZRsmc/189aTH1IYb4fzFlQRK/0FL517KAOHT81fyhJBnMWRqcX4LWV27j/zXVcnD+Urx2ZwxMLixP2/6Jzjl/8YwUrt1Twh6njCKQkJfyg6qqaOp5fsoWzjx1EpjcBoSSGWfM1O7iIdD++DwAAjh2aye1TxvDeup3c/dqaeDdHYqy8qpaXl2/lvPFDyAgkd+gcgzIz+PpR/XlyYTF19aGI2rNh515ufGIpxw7J5NdTxjCjIJdtFdW8/emO1g/uhh6fv4mnCou5/pQRXDhpKGeNGcizizezryZxB1W/vHwblfvrmFbQ8mRzItK1qAqQJCoFAJ6LC3KZMTmPB9/6jFdXbIt3cySG/rFkM/vrQhHfjE0vyGVH5X7eXFPS4XNU1dRx1d8LSUk2HrxsIumpyZx69AD69Qwwe37iPRVfvKmU255fyclH5vDD044EYHpBHpXVdbyyontNANses+dvYni/Hkw+vE+8myIiHZCY/bHiZwoAGrnt3GMYl5vFT55c6qtKLH7inGPW/E0cOySTMUMyIzrXN0b1J6dXGnM6OBjYOcdNTy9n7Y5K7psxgaHZ4ZmIAylJXDhxKPNW72BHRXVEbexKdu7Zz9WPLmJgZjp/nD6e5KTwo7Xjh/dhWN9gQgY8AOt2VLLw81KmFeRiepwo0q3oN1YSlQKARtJSknnw0omkpSRx1aOF7NlfF+8mSZQtKy5n9bbKqKRipCYnMXXSUP65egfbytt/o/4/72/khaVbuPH0ozhpZM4B26YV5FIfcjy1qDjidnYFdfUhrnt8EaVVNTx46SSygoEvtpkZ0wrymL9xN5+VJF7gPXt+ESlJxoWThsa7KSLSUSoDJAlGAcBBBmdl8KcZE1hfsoefPrVU038nmNkLNpGRmsyU8YOjcr5pBbmEHDxV2L6n1x+v38VvXl7F6ccM4JqvH3HI9uE5PZl8eB/mLCgilACDgX83dw0frd/Nb84/tsmelwsnDSElyXgiwQYD76+r55nFm/nmMQPo1zMt3s0RkXYK6R5AEpQCgCacOKIfPz1zFC8v38Zf3t0Q7+ZIlOzd71ViGTuIXunRqcRyWN8enHhEX+YsbPuN+vaKaq59fDGH9Qnyh4vHNZsWMmNyLp/vquKjDbui0tZ4eWnZVma+s57Ljz+s2afg/Xulc+rR/XmqsJiausgGVXclr3+ynd17a5g+uf3lZkUk/q55bBGgMQCxNGv+Jq55rDDezfAdBQDNuPJrwzlrzEDufHU1H37WvW/AJOzFZVvYW1PPjMnRrcQyrSCXot37+KAN/09q6kJc/WghVTV1PHz5pBYDkbPGDKJ3ekq3zo1fu72Sf39qKRPzsviPbx/T4r7TC/LYtbeGeauiP8NyvMxZUMSQrAy+OqJfvJsiIh2wvULzPMTaLc8s5+XlKr7S2RQANMPM+P3UcQzrG+S6xxextXxfvJskEZq9oIgR/XsyMS87quc9Y/RAsoKpzGrDYODbX/qERZvK+P1F4xg5oFeL+6anJnP+hCG8umIbpd1wkrrK6lqu/HshwUAyD1w6iUBKy39uvnZkDoMy05mVIGlARbureHftTi7Oz/1iwLOIdE/KBJJEowCgBT3TUnj48nyqa+u5+tFF7K9L3DrliW7NtkoWbypjegwqsTTcqL+2cluLs0k/s6iYv334Od8/6XDOHtu2CcimFeRRUx/i2cWbo9XcTuGc4ydPLuXz3VXcf8lEBmamt3pMcpIxNT+Xd9eWUFwavRmW4+WJhUUkWXjSOBERka5EAUArRvTvyd1Tx7GkqIxfv/BJvJsjHTR7wSZSk40LJsbmZmx6QR619Y5nmqnas3JLObc8s5zjh/fhpjNHtfm8xwzuzbihmcxZUNStBqQ/+PZnzF25nVvOGsXxw/u2+biLvZvlJxZ27+pHdfUhnlhYxMlH5jA4KyPezRGRCDmNAoi58x94PyGKXnQXCgDa4KxjB3HlycN57ONNPLkwMdIT/KS6tp5nF2/m9NED6dMj0PoBHXDUwF5MyMtidhM36mVVNVz1aCHZwQD3XzKRlOT2/dpNK8hjzfZKFheVRbPJMfPe2p3cPXcN3x47iO999fB2HTs0O8hJI3N4cmER9d34g+DtT0vYXrGfaQUa/CuSCFZsroh3ExLe4k1lVNUq06KzKABoo38//ShOPKIvP//HClZsLo93c6Qd5q7cRllVLTNifDM2oyCPdTv2UPh56RfrQiHHj+YsYVt5NQ9cNrFDpSDPHT+YYCCZOd1gMHBxaRU/mLWIEf17cteFYzuUbjWjIJet5dW882nHZ1iOt1nzi+jXM41Tj+4f76aISJS83Y3/JokcTAFAG6UkJ/GnGRPo1yPAlX8v7JaDMv1q9vwicvtkcOIRbU9F6Yizxw6iRyCZ2Y0Gsd47by1vrSnhl+eM7vDg455pKXx77CBeWLalS09OV11bzzWPLaKu3vHQZZPokZbSofOcevQA+vYIMLuDMyzH2/aKat5cs4OLJg0ltZ29PZI43l+3M95NkChLpJnZRSL6dDKz35vZajNbZmbPmllWo223mNk6M1tjZmdE3tT469szjQcvm0RJ5X6un724W6co+MXGnXv5cP0upuXnkhTjSiw90lI4d/wQXly2hYrqWuat2s5989Zy4cShXHZcZL0P0yfnUVVTzwtLt0SptdF32/MrWVZczh8uHsfwnJ4dPk8gJYmLJg1l3qod7Kjsfh+4TxUWUx9yUZltWrqvnz61LN5NkCjTJ74kkkgfT70OjHHOjQU+BW4BMLNjgOnAaOBM4AEzS47wWl3CuNwsfjVlNO+u3ck9r38a7+ZIK+Z4lVgumtQ5N2PTC3Kprg1x/z/X8aM5Sxg9uDd3nD8m4spDE3KzOHJAzwN6F7qS2fM3MXtBEdd+4whOHz0w4vNdXJBLXcjxdGH3qn4UCjnmLCji+OF9OLxfj3g3R0SiSRGAJJCO9dF7nHOvNVr8CLjIez8FmO2c2w9sMLN1wGTgw0iu11XMmJzHkk1l3P/mOrKCqeT2CXboPONzsxjQu/XyiNG0Yede+vYM0DtKM+G21eptFRzerwdpKZ0XB9bWh3iqsJhTRvVvUxnKaBg7NJOjB/Vm5jvrycxI5aHLJpGeGvnXbGZML8jj1y9+wqqtFRw9qHcUWhsdS4vK+OVzKzlpZD9u+OZRUTnnETk9mXx4H+Ys2MRVJw+PeunWWPlo/S427a7ixtOPjHdTJM66U9UuaRtVApJEElEAcJB/A+Z474cQDggaFHvrDmFmVwBXAOTldZ+KGb+aMprV2yu5/aVVHT5Hv55pvHT9VzstCFixuZwLHvyAETk9efrqE8kIdM7N+CvLt3L1Y4s4Y/QAHrpsUqfdzD301meUVO5nxuTO+39lZvzLiYfxs2dXcN+MCR0ODpty/oQh3PnKauYsKOK2c0dH7byR2LVnP1c/WkhOrzTumz4hqhNeTS/I5YYnlvLR+t2cEOPxG9Eya0ERmRmpnBGFXhDp3nSrmHgU00kiaTUAMLM3gKY+zX7unHvO2+fnQB3wWMNhTezf5K+Oc24mMBMgPz+/2/x6pacm8+SVJ7B2R2WHjt+1J1wa8prHFjHr+8e3OktqpEr31nDl3wvplZbCqm0V/Pwfy/nD1HExvxlft6OSnzy5lH4905i7cjsPvv0Z13x9REyvCfDWmh381xufMmX8YE4Z1bmVWKYV5HHmmEFkZkS3lyW7R4AzxwzkmUXF3HzWqKj0LESirj7E9bMXs3NvDc9cfSLZUS6x+q1jB3Hr8yuZs2BTtwgASvfWMHfFNi45Li/uPxuJv63l3W/8irSs29ygdGPdo683MbQaADjnTmtpu5l9F/g2cKr7ss+zGGicdD0U6LqjFzsokJLE6MGZHT7+9xeN49rHF3HHS5/wqyljotiyA9WHHNfPXkxJ5X6evOoE3lyzg3vfWMuE3CwuP2FYzK67Z38dV/69kIxAMi/84Cvc8dIq7p67hrFDsvjqyH4xu27R7ip+OHsJRw3oxW8vODYu6SPRvvlvML0gl+eXbuHVFds4b0KTnWqd5u7XPuX9dbv43UVjGTOk478HzWmYYXn2giJuq6ohKxibORyi5ZnFm6mpD2nwr0iCUg9A7Olb3HkirQJ0JnATcK5zrqrRpueB6WaWZmaHAyOB+ZFcKxGdPXYQ3z/pcB758HOeXRy7mU/vef1T3l27k19NGc243CyuP2Ukp4zqz69f/OSAmvXR5Jzj359cysZdVfxpxkQGZWZw14VjGdG/Jz+YtYji0qrWT9IB1bX1XPn3QkIuXIoyGIhmllv8HT+8L4f1DTJrfnxLZL66YisPvf0ZlxyXx8X5sbvhnV6QR01diH8s7tqDgZ1zzFmwiXG5WV1qfIaIdFwsP5dF4i3SvJP7gV7A62a2xMweAnDOrQSeAD4BXgWudc5percm3HTmKI47vA+3PLOcT7ZEf6bB11Zu4/431zEtP/eLXPikJOOei8czKDODax4rjEmpxYffWc8rK7Zx85mjvkjf6JGWwsOX51NX77j60UVUR3nGP+ccP392BZ9sreDeaeMZloBVWJKSjIvzc/l4w27Wl+yJSxvW7djDjU8sZXxuFreec0xMr3XM4N6MHZrZ5AzLXcmiTWV8un0PM/T0XyQhLNpUyo/nLD1gnQYBS6z9ad5aht38EkW7Y/OQtLGIAgDn3AjnXK5zbrz3uqrRtjucc0c4545yzr0SeVMTU0pyEvdfMpGsjABXPVpIeVVt1M69viR8ozZ2aCa/mnLgoNHMYCoPXz6J8n21XPf4YmrrQ1G77vvrdvK7V1dz9thB/L+TDj9g2+H9evBf08azfHM5tz63MmrXBHj04008vaiY608dyalHD4jqubuSqZOGkpxkzFnY+SVBw2ldC0lPTebByyZ2SlWn6QV5rN5WydLirjsD95wFmwgGkvn2uMHxboqIREHV/kMfUJVU7o9DS8Qv6upD/MErL//qim0xv56mqewCcnql8cBlE9lavo8fzVlMKAoTjO3dX8dVjxaSkmw8cOnEJgclHj2oN3deMJb5G3Zz5yurI74mwOayffxg1mKOyOnJ7y4c22T+/TePGcB13xjBnIVFUUtlKfy8lF+/sJKvH5XDj04dGZVzdlX9e6dzyqj+PF1YHNXArTUNaV0bdu7lT5dMYFBmRqdc95xxg8hITWZ2nNOemlNZXcsLS7dy7rjB9Ozg7Mci0rU0VdDs3jfWdn5DxDfeWlPSqddTANBFTMzL5pfnjObNNSX8cV5kf2Scc9z09DLW7djDn2ZMZGh286Uoz5swhH85cRh/fW8Dz0c4y2x1bT3XPFpITV2Ihy6fRI8WboZ+/M0j+dqROdz63EqWFJVFdN2Syv1c81ghAzPTuXfa+JjP+NsVzJicy849Ncxbtb3TrjmzIa3rrFGceETsBnEfrFd6KueMG8TzS7ewZ39dp123rV5YupV9tfUa/CuSSBL/Y6RL0re98ygA6EIuOy6PCycO5Y/z1vLP1R2/sfvrext4cdlWfnLGUW2qtvOzbx1N/mHZ3PTUMtZs61hZU4BfvbCSpcXl/OHicRyR07PFfZOTjD9OG0//3mlc82ghu/Z0rGu1rj7EdY8voqyqlocum9TlK8VEy9dG5jCwd3qnzQz8wbqd3PXqas4+dhDfP2l4p1yzsWkFeVTV1PNihEFqLMxesIlRA3sxPjcr3k0RkShJ6iaTD3Z3++s0PLRBUqM78s4Yb6IAoAsxM+44fwyjB/fmR7OX8Pmuve0+x0frd/HbV1ZzxugBXH3yEW06JpCSxAOXTqRnegpXPVpIRXX7xyHMWbCJWfOLuObrR7R5EqTsHgEeumwSu/bW8INZi6nrQDrLna+s5uMNu/ntBcdGVJK1u0lJTmJq/lDe/rSEzWX7YnqtLWX7uG7WYobn9OSui5pO64q1iXlZjOzfs9MCnrZauaWcZcXlTCvI7TazFYtI6/Tb3DnO+dN78W5Cl/HDWUs69XoKALqY9NTkL2bLvfLvheyraXt0vK28museX8RhfYLc3c5Jvvr3TueBSydStLuKG+Ysbdc4hGXFZfzHcys5aWQ/bjz9qDYfBzBmSCa3nzeGDz7bxe9fW9OuY19ctoW/vLeB75xwGBdMHNquYxNBQ/nNJ2M4GHh/XT1XP7aImroQD18+KW457mbG9Ml5LCkqY/W26FfL6qgnFhQRSEni/DjPySAi0VW4KTYlsuVAn24/sJqdn+ssVTZKcbVOCEEVAHRBuX2C3DdjAmu2V3LLM8vaVP6wpi7E1Y8VUlVTz8OXT6JXevsnoioY1oeffeto3li1nQfeWtemY3bvreHqRxeR0zONP06fQHIH8u+n5udyyXF5PPz2el5ZvrVNx3y6vZKfPrWMSYdl84uzY1uKsqvK7RPkqyP68cSCIuqjMHC8Kbc9/wlLi8q4e2rraV2xdv6EIQSSk5g9v2v0AlTX1vPs4s2cNWagb1LPRPzid682/UDqz2+27bOxu3ro7c849349lfcDBQBd1MlH5nDDaUfyjyVbeOSDja3uf/tLn7B4Uxm/v2gcIwf06vB1//Urw5gyfjB/eP1T3vm05RHp9SHH9bMWU7JnPw9dNok+PTp+E3TrOccwPjeLnzy5lHU7Wq5vX1Fdy1V/LyQYSOGBSycSSPHvf+PpBXlsKa/m3bXRrx7wxIJwlaarv34EZ45pW1pXLPXpEeCMMQN5dvHmqM8h0RGvrNhKRXUd0wvy4t0UiZN1Ozo+Zkq6p8c++jzeTYipO19ZzbI4llwOdeH5XjqTxgD43LXfGMFpR/fn9pdWsWDj7mb3e2ZRMX/78HO+f9LhnD12UETXNDN+e8GxHDWgF9fPXtziZBR3v7aG99bt5PYpYzh2aGT592kp4bry6anJXPn3hc1WewmFHD95Yimf767iz5dMYEDv9Iiu292ddkx/+vQIRP2p+LLiMn7x3Aq+OqIfP2lnWlcsTS/IpXxfLXNXxr5GcmtmzS9iWN8gxw/vE++mSJxc38k5u4lue0U1KzZ33fk+AGrqdYMaS2Nve63Vh4ASHQoAurCkJOMPF49naHYG1zy2iB0Vh87Yu3JLObc8s5zjh/fhpjNHReW6wUAKD102ifqQ4+rHCpt82vrqim08+NZnzJicx8VRKn84KDODP10ygQ079/LvTy5tMvXpwbc/47VPtvOzbx3NccP7RuW63VlaSjIXThzCG6u2R22SmsZpXffN6FhaV6ycMLwveX2CcU8DWl+yh/kbdjOtIE+Df33sk60tj0cZdvNLvLis61Wu6qq+/vu3+HYXHxS6s4MV6+RQzU12tXxzZKXBpW0UAHRxmRmpPHT5JPZU13Ht44sOmPiprKqGqx4tJDsY4P5LJpKSHL0f57B+Pbh32nhWbK7gP/6x4oCb8XU79vCTJ5cyLjeL286Nbv79iUf04+azRvHKim3MfGf9AdveXVvCH15bwznjBvNvXxkW1et2Z9MKcqkLOZ5eVBzxuRqndT142cSI0rpiISnJmFaQy4frd7FxZ/urZEXLnAVFpCQZF07S4F9p2XWPL453E7qNfd7DpkjKUUdq4869XP1oYYv7tGVcnrTu0+1N/5x/PGdpJ7fEnxQAdAOjBvbmzguPZcHGUu54aRUQToX50ZwlbCuv5oHLJtKvZ1rUr3vq0QO4/pQRPFlYzOPeLKx7vBmG01KSePDSiaSlHDrDcKS+f9Jwzj52EHe9upoP1u0EoLi0iutnLWZk/17cdeGxeurayIj+vSgYls2cBUURfzD9wUvr+s8poxk7tGvWtb9o0lCSk4w5Max+1JKauhBPLyrm1KP707+Xv1PQosXMzjSzNWa2zsxujnd7WrJnf127x6CUV7W/tHI8fbR+V6c96d5RWU1ldS2LG1XdOePed6J6jfqQo6qmbZMIfv3ut3ilmSfTDe58dXU0mtWlbW8i4yDadu+tifk1uou3WxlzGQsKALqJKeOH8K9fGcb/fbCR55Zs5t55a3lrTQm3njOaiXnZMbvuD087kpOPzOG251eyaFMpP31qKetL9vCnGRMYnJURk2uaGXddNJbhOT25btZiNuzcy9WPLqKu3vHQ5ZMIBuJTirIrm1aQx4ade/l4Q/NjRVozd+U2HnjrM2ZMzmVaFx7YOqB3Ot84qj9PLiw+oEess8xbtZ2de2o0+DdKzCwZ+DNwFnAMMMPMYlbaa0lRGQ+8tY7SvTXMnr+JmroD/w+FQo7yfV/esFfX1n/R21S6t4Yxt85l0n++zsx3PmvzNcf9+rUuMXC9pi7EEwuLmPnOZ4d83ZXVtWzcuZfL//ox02d+RP7tbxxy/H+9/iln3HPgzflf3l3PmR24YX9z9Q6KS6uYfMc8jr3tNc5/4IMDtjd1c7h7bw3THv6QF9owIeDNTy/juSWbefvTEo742csc88u5PF3Yci9paRtvSB/7aFOb9uuKdlRUs6Oymt17aw4J8hpPyHncb+axPIaDgffV1PN/LRQ4iWV569Y455pMuY5EbX2IymbmWNq9t4brHl8U1eu1hXWlrqz8/Hy3cOHCeDejy6qtD3Hpf3/MkuIyaupCXDRpKL/vhImZyqpqOOf+9yip3E91bYhbzhrFlW2cZCwSn5XsYcr971MXClFdG+Iv38nntGMGxPy63dG+mnom3/EG+cOyueS4w9p/fG09P3tmOUfk9OCJq06ISc9ONM1btZ3vPbKQG795JKMG9e7Ua8985zOKS/fx3k2ndPr4CDMrdM7ld+pFY8zMTgBuc86d4S3fAuCc+21T+3f0c2LF5vJm88tTk43aVgZ3Ds5MZ0t5ZDcFXzsyh/G5Wdw3b+0B688dN5jnl27huMP7sKV8H18/sj/vri1h464vizCcNLIfvdJT+HxXFSu3tDz24FvHDmT04Ez27q9je8X+iNIDTxjel0BKUqtPKBu+hoMdOyST5Y0G9n5lRF/eX7erzdcfn5vFkqKmc8LvmTaOnZXhG9mHD0oZbc2ogb3YVlFNmdc7k5Jk1LWznHJGajKnjx5AwbA+5PQK98I7B4s2lfL4x5vo2zPAjMl5JJtxWN9geHuj4z/ftZd+PdPYWl7NQ29/xiWT85iQl83dr62hpi7EJq8IR7+eaVxyXHgelDXbKhjYO52Nu6oYOzSTQHIS81bvAOC4w/uQ1yfIJ1sr2LBzL1WN5hEyC7etOclJ1mQ56Ql5WRiwcksF3ziqP8G0ZJ5ZtBmAQZnpjBuaxfqde76o53/a0QMY0b8nkw7L5sPPdvH4/M+prg0Hmz0Cyfz2wrFcP6vtaXGTDsum8PNShmRlMDV/KIaR0yuNmrp6Hvnwczbs3MuJR/SlLuTI6ZVGvx4BPt6wm9WN0sjOHD2Q99bt5Csj+jJ35XZGDez1xfYxQ4Z80MAAAA6OSURBVHqzYnPLv0/TC3KZmJdNRXUtS4rKqK4NsaOy+oBqSd854TAWbCxlYO803lzz5e/KjMm51NR9maI7NDuDn545qtXvwfWnjOCGDhTgaM9nhAKAbmZHZTXn/Ok9cnr9//bONUaS66rjv9Pv6Xn2zNje2d3ZV2SDHEjih4wTkmAw+AXEQAJaQMQiIMsJNkRRBEaWIotvIYIPEBSLgJUEGWJsCKyQTWKBAQmw4/ixzi7OZsfrXXu9s7szszM90+/X5UPVTNq9VdPVPVXVPdPnJ7Wmuu4t3f+ce+49dapuVSd58r4PkIqHc6J2/FyWj37pf7j1h6/ii792XWhLcP712Hk++diLPPCT3Q2GQeLhI8c3vaLSjqnhBEce+CB7Arqz4ye1eoOf+MJ/BP4ryG585meu4XdvvTr0dndoAvAx4A5jzG/b338D+DFjzP1Nde4F7gXYt2/fDWfOdP4qxs8+cZQn21wBVhRF6Qc+dcu7+P0uXuyiCcAOZ61UJRGLhH6VdjlfYXwoTiTkq55LuTJTATzjsNMo1+qcvND969NmM2nG053/gFyvWMqVmd/iFdluiIhwzVUjvj5075UdmgD8MnB7SwJwkzHmAaf63caJt1eKnFnMcy5bIhWPEBVhPB2n3jA0DGSLVRLRCKOpGG8s5tk9kaJSMyzly0wNJxgfSvD6Qg4D7JtMc2G1RDIWIVeucdOByY3fX1nMlTm3UiQiwsJamYYxLBeqZNJxxobiVOsNzl4qkivXNn5Ze3YyzXy2yK7xFAtrZa4YSbJcqBKNCLVGg/PZEtfOjFGo1KnUG5RrdYbiUUrVBvlKjV1jKd5YzDM2FGcynWByJEFEhGq9sdGewbBSqHLFaJJyrcGh6WFOL+WpN6z13tMjCUrVBqVqnf3TwwwnoiRiEYyBWqPBpXyVUrXOzHiKbLFKKh5lKVdmKBHjwFSafKXOSqHC5HCC+WyJdCLKbCZNtlglV65RqNS5cjRJOhHl1GKeWt0wkY6TLVbZmxkiW6xiDPz33CI/+54ZVos13lou8CO7x3nu1BL7ptKUaw2uHE2yVqqRiEU4ND1MrlyjWm+wWqwxny1ycHqYVDxKqVonGYuyUqwQiwgREcq1BsuFCu/ePcbbKyXGUjFOL+ap1g03Hsjw7IkFdo+nmB5JslauMj4UZ3GtAmJd7X7rUpGZ8RSZ4QRnlwsMxaOkEzGiESGdiLJ+XWwxV+F8tshwMsaeiSHy5ToTTXPrer1CpU48GsEYw8tvrnDNVaNMpOP837lVkvEIZfvKeWY4QSxqHbRarDI5nODUQp5d4ymSsQhnlgpMpOOMpeLUGoZavYHBemHH9EiSTDrOSCrGUq7CSqFCNBphZixFpd4gV64xmoxRaxhy5Rr5co29mSGS8SjlaoOxVIyFXJm1Uo39U2kiIrz85jL7p4aZHE7QMIZCpc6J82vcsD9DrlxjeiRh2b5Q5VKhwpnFPJnhBPsm00wOJ3jzUoFcucZSrsL0SIJ4NMJ7ZyfIlWpcXCuRjEW5sFoiFhUy6QSnl/Icmh4hFrX6cGIoTkSE+WyRE+fXeM/sxMb5STIe4dxKiXhUWM5X2DWeYjQVZ7lQYSwVZ+5ijr2ZIap1Q0QgnYxxaiFHLCKk4lFEhHKtTsO+o5AtVrl2ZhwRWMiVqdYaxGMR6nVDtd7g/GqJfZNpazzkK+yZSPHC6WVmM+kNX1spWsvrDkwPM5KMMRSPcnGtRKFSp1ipMz4U59WzWX507zixiDA1kuR9s909g6cJgKIoyg5khyYAoSwBUhRF2el0EiP0IWBFURSll7wAXC0iB0UkARwGjvRYk6Ioyo5GX6eiKIqi9AxjTE1E7ge+CUSBR40xx3ssS1EUZUejCYCiKIrSU4wxTwFP9VqHoijKoKBLgBRFURRFURRlgNAEQFEURVEURVEGCE0AFEVRFEVRFGWA0ARAURRFURRFUQYITQAURVEURVEUZYDQBEBRFEVRFEVRBghNABRFURRFURRlgBBjTK81bCAiC8CZLg+fBhZ9lOMX/ahLNXmjHzVBf+pSTd7Ziq79xpgr/BSz3diBcaIfNUF/6lJN3ulHXarJO93q8hwj+ioB2Aoi8h1jzI291tFKP+pSTd7oR03Qn7pUk3f6Vdcg0I+270dN0J+6VJN3+lGXavJOGLp0CZCiKIqiKIqiDBCaACiKoiiKoijKALGTEoC/7LUAF/pRl2ryRj9qgv7UpZq806+6BoF+tH0/aoL+1KWavNOPulSTdwLXtWOeAVAURVEURVEUpT076Q6AoiiKoiiKoiht2HYJgIjcISInRGRORB50KE+KyON2+fMiciBgPbMi8qyIvCYix0Xk9xzq3CIiWRF5xf58LkhNTe2eFpHv2m1+x6FcROTPbFu9KiLXB6znh5ps8IqIrIrIp1vqhGIrEXlURC6KyLGmfZMi8oyInLT/ZlyOvceuc1JE7glY0xdE5Ht2/3xDRCZcjt20r33W9LCIvN3UR3e5HLvpWPVZ0+NNek6LyCsuxwZlJ8d5oNc+pfyAoPzRpS03f3AdOyLyh7a2EyJyexC6nfzfzUc3iw9++atbTOiFnTqJCd3YRkRusG0/Zx8rXWpyjAkickBEik02e6Rd217nJw+afOsvETko1rnbSbHm9UQ7TZvocowLIdqqo7gQll9tYIzZNh8gCrwOHAISwFHg2pY6nwIesbcPA48HrGkGuN7eHgW+76DpFuBfemCv08D0JuV3AU8DAtwMPB9yX57Hemdt6LYCPgxcDxxr2vfHwIP29oPA5x2OmwRO2X8z9nYmQE23ATF7+/NOmrz0tc+aHgY+66F/Nx2rfmpqKf8T4HMh28lxHui1T+kneH/s0B8cx45ddhRIAgdtrVG/dTv5v5uP4hIfgvJXmmJCL+zkMtf5Zhvg28D77WOeBu7sUpNjTAAO4D4nOrbt9v91ocm3/gL+Hjhsbz8CfLLb/msp34gLIdqqo7gQll+tf7bbHYCbgDljzCljTAX4OnB3S527ga/a208Ct3aUEXWIMWbeGPOSvb0GvAbsCao9n7kb+JqxeA6YEJGZkNq+FXjdGNPtD/psCWPMfwGXWnY3+85XgV9wOPR24BljzCVjzDLwDHBHUJqMMd8yxtTsr88Be/1oayuaPOJlrPquyR7rvwL8nR9tdaDJbR7oqU8pGwTmj050ERfuBr5ujCkbY94A5mzNYeh281G3+BCUv3qJCYHZqcOY0JFt7LIxY8z/Guus7Ws4zwVtNXUaE9q07WV+aqtpEzrqL3v+/imsczfPmtrp8hoXArBVp3EhFL9aZ7slAHuAt5q+n+XySXWjjj1IssBUGOLEWm50HfC8Q/H7ReSoiDwtIu8OQw9ggG+JyIsicq9DuRd7BsVh3AdjL2wFcJUxZh6sgQtc6VCnlzb7BFaG70S7vvab++1blI+63ArtlZ0+BFwwxpx0KQ/cTi3zQL/71KDQMxs7xAWnseOmz2/dTv7v5qNhaVqnNSb00k7r+GWbPfa23/paY8JBEXlZRP5TRD7UpNWtbS/zk1f86K8pYKUpwfHLTk5xIVRbeYwLofrVdksAnK7kt77GyEsd3xGREeAfgE8bY1Zbil/CWuryXuDPgX8KWo/NjxtjrgfuBH5HRD7cUt4rWyWAjwBPOBT3ylZe6ZXNHgJqwGMuVdr1tZ98CXgX8D5gHuvWais9sRPwq2x+lSdQO7WZB1wPc9inr2fzl36JC25jx02f37o78f+wNDnFhF7bqR2d6gjCZq0xYR7YZ4y5DvgM8LciMhZE2w741V9BaW2NC6HaqoO4EKq9tlsCcBaYbfq+FzjnVkdEYsA43S1h8IyIxLE69zFjzD+2lhtjVo0xOXv7KSAuItNBarLbOmf/vQh8A+u2WzNe7BkEdwIvGWMutBb0ylY2F9aXQNl/LzrUCd1m9gM/Pwf8un2b7zI89LVvGGMuGGPqxpgG8GWXtnphpxjwS8DjbnWCtJPLPNCXPjWA9MIfL/OHTcaOmz5fdbv4v5uPhqLJ5h0xodd2asIv25zlnUt1tqTPKSbYy2yW7O0XsdbYX9OmbS/zU1t87K9FrGUvMQetXeEUF8K0VYdxIVS/2m4JwAvA1WI9JZ7AumV4pKXOEWD9CemPAf/udtLkB/basr8GXjPG/KlLnV3rzyGIyE1Ydl8KSpPdzrCIjK5vYz04dKyl2hHg4/aT5zcD2fXbUgHjepW2F7Zqotl37gH+2aHON4HbRCRj3+a8zd4XCCJyB/AHwEeMMQWXOl762k9Nzc+J/KJLW17Gqt/8NPA9Y8xZp8Ig7bTJPNB3PjWghOqPbv6wydg5AhwW6y12B4GrsR7u8033Jv7v5qNu8SEIf31HTOilnVrwxTZ22ZqI3Gz7xsdxngva4hYTROQKEYna24ewbHOqTdte5icvmnzpL/tc7Vmsc7ctaWrisrgQlq26iAvh+pXZ4pP7YX+wnpL+PlbG9pC974+wBgNACus24hyWox0KWM8HsW65vAq8Yn/uAu4D7rPr3A8cx3rS/TngAyHY6ZDd3lG77XVbNesS4C9sW34XuDEEXWmsE/rxpn2h2wor2MwDVaws+rew1h/+G3DS/jtp170R+KumYz9h+9cc8JsBa5rDWvu37lvrb7jaDTy1WV8HqOlvbH95FWvCmmnVZH+/bKwGpcne/5V1P2qqG5ad3OaBnvqUft7RR4H4Y4f+4Dh27GMesrWdoOlNHn7pdvP/TXzUNT746a84x4TQ7eQy1/lmG3vMH7OP+SJYP8TahSa3mPBRfhA7XwJ+vl3bbv9fF5p86y/bT79t/59PAMlu+8/e/xUujwth2arTuBCKX61/9JeAFUVRFEVRFGWA2G5LgBRFURRFURRF2QKaACiKoiiKoijKAKEJgKIoiqIoiqIMEJoAKIqiKIqiKMoAoQmAoiiKoiiKogwQmgAoiqIoiqIoygChCYCiKIqiKIqiDBCaACiKoiiKoijKAPH/aR8+nqoKDuUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_frames = 20000 # 1400000\n",
    "batch_size = 32\n",
    "gamma      = 0.99\n",
    "\n",
    "losses = []\n",
    "all_rewards = []\n",
    "episode_reward = 0\n",
    "\n",
    "state = env.reset()\n",
    "for frame_idx in range(1, num_frames + 1):\n",
    "    epsilon = epsilon_by_frame(frame_idx)\n",
    "    action = model.act(state, epsilon)\n",
    "    \n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    replay_buffer.push(state, action, reward, next_state, done)\n",
    "    \n",
    "    state = next_state\n",
    "    episode_reward += reward\n",
    "    \n",
    "    if done:\n",
    "        state = env.reset()\n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        \n",
    "    if len(replay_buffer) > replay_initial:\n",
    "        loss = compute_td_loss(batch_size)\n",
    "#         losses.append(loss.data[0])\n",
    "        losses.append(loss.item())\n",
    "        \n",
    "    if frame_idx % 10000 == 0:\n",
    "        plot(frame_idx, all_rewards, losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
